Repository: aptos-labs/aptos-core
Pull request #16313: [aptos-framework] Remove balance checks in DFA
Author: login: runtian-zhou
Created: 2025-04-09T17:04:46Z | Merged: 2025-04-17T20:10:03Z

## Description
This PR removes the balance check in the `dispatchable_fungible_asset` module that was verifying the withdrawn amount matched the requested amount. The check was unnecessary and was causing tests to fail. Additionally, it enhances the `ten_x_token` test module by implementing and registering proper withdraw and deposit dispatch functions.

## How Has This Been Tested?
The changes have been tested through the existing `nil_op_token_tests` which previously expected a failure but now passes correctly. The test verifies that withdrawal and deposit operations work properly without the balance check.

## Key Areas to Review
- Removal of the balance check in `dispatchable_fungible_asset.move` which was comparing start and end balances
- Implementation of withdraw and deposit functions in the `ten_x_token.move` test module
- Registration of these functions in the constructor of the test token

## Type of Change
- [x] Bug fix
- [x] Refactoring

## Which Components or Systems Does This Change Impact?
- [x] Aptos Framework

## Checklist
- [x] I have performed a self-review of my own code
- [x] I have tested both happy and unhappy path of the functionality
- [x] I have made corresponding changes to the documentation


Diff:
diff --git a/aptos-move/framework/aptos-framework/doc/dispatchable_fungible_asset.md b/aptos-move/framework/aptos-framework/doc/dispatchable_fungible_asset.md
index bfe682fe63c..7dc2ce54d7a 100644
--- a/aptos-move/framework/aptos-framework/doc/dispatchable_fungible_asset.md
+++ b/aptos-move/framework/aptos-framework/doc/dispatchable_fungible_asset.md
@@ -228,7 +228,6 @@ The semantics of deposit will be governed by the function specified in DispatchF
             <a href="../../aptos-stdlib/../move-stdlib/doc/features.md#0x1_features_dispatchable_fungible_asset_enabled">features::dispatchable_fungible_asset_enabled</a>(),
             <a href="../../aptos-stdlib/../move-stdlib/doc/error.md#0x1_error_aborted">error::aborted</a>(<a href="dispatchable_fungible_asset.md#0x1_dispatchable_fungible_asset_ENOT_ACTIVATED">ENOT_ACTIVATED</a>)
         );
-        <b>let</b> start_balance = <a href="fungible_asset.md#0x1_fungible_asset_balance">fungible_asset::balance</a>(store);
         <b>let</b> func = <a href="../../aptos-stdlib/../move-stdlib/doc/option.md#0x1_option_borrow">option::borrow</a>(&func_opt);
         <a href="function_info.md#0x1_function_info_load_module_from_function">function_info::load_module_from_function</a>(func);
         <b>let</b> fa = <a href="dispatchable_fungible_asset.md#0x1_dispatchable_fungible_asset_dispatchable_withdraw">dispatchable_withdraw</a>(
@@ -237,8 +236,6 @@ The semantics of deposit will be governed by the function specified in DispatchF
             <a href="dispatchable_fungible_asset.md#0x1_dispatchable_fungible_asset_borrow_transfer_ref">borrow_transfer_ref</a>(store),
             func,
         );
-        <b>let</b> end_balance = <a href="fungible_asset.md#0x1_fungible_asset_balance">fungible_asset::balance</a>(store);
-        <b>assert</b>!(amount &lt;= start_balance - end_balance, <a href="../../aptos-stdlib/../move-stdlib/doc/error.md#0x1_error_aborted">error::aborted</a>(<a href="dispatchable_fungible_asset.md#0x1_dispatchable_fungible_asset_EAMOUNT_MISMATCH">EAMOUNT_MISMATCH</a>));
         fa
     } <b>else</b> {
         <a href="fungible_asset.md#0x1_fungible_asset_unchecked_withdraw">fungible_asset::unchecked_withdraw</a>(<a href="object.md#0x1_object_object_address">object::object_address</a>(&store), amount)
diff --git a/aptos-move/framework/aptos-framework/sources/dispatchable_fungible_asset.move b/aptos-move/framework/aptos-framework/sources/dispatchable_fungible_asset.move
index 40087d979bf..293b4e069ef 100644
--- a/aptos-move/framework/aptos-framework/sources/dispatchable_fungible_asset.move
+++ b/aptos-move/framework/aptos-framework/sources/dispatchable_fungible_asset.move
@@ -84,7 +84,6 @@ module aptos_framework::dispatchable_fungible_asset {
                 features::dispatchable_fungible_asset_enabled(),
                 error::aborted(ENOT_ACTIVATED)
             );
-            let start_balance = fungible_asset::balance(store);
             let func = option::borrow(&func_opt);
             function_info::load_module_from_function(func);
             let fa = dispatchable_withdraw(
@@ -93,8 +92,6 @@ module aptos_framework::dispatchable_fungible_asset {
                 borrow_transfer_ref(store),
                 func,
             );
-            let end_balance = fungible_asset::balance(store);
-            assert!(amount <= start_balance - end_balance, error::aborted(EAMOUNT_MISMATCH));
             fa
         } else {
             fungible_asset::unchecked_withdraw(object::object_address(&store), amount)
diff --git a/aptos-move/framework/aptos-framework/tests/clamped_token.move b/aptos-move/framework/aptos-framework/tests/clamped_token.move
new file mode 100644
index 00000000000..d5994bb298c
--- /dev/null
+++ b/aptos-move/framework/aptos-framework/tests/clamped_token.move
@@ -0,0 +1,92 @@
+#[test_only]
+module 0xcafe::clamped_token {
+    // Create a token with max amount one can withdraw on each withdraw call.
+
+    use aptos_framework::fungible_asset::{Self, FungibleAsset, RawBalanceRef, RawSupplyRef, TransferRef};
+    use aptos_framework::dispatchable_fungible_asset;
+    use aptos_framework::object::{ConstructorRef, Object};
+    use aptos_framework::function_info;
+
+    use std::option;
+    use std::option::Option;
+    use std::signer;
+    use std::string;
+
+    struct BalanceStore has key {
+        balance_ref: RawBalanceRef,
+        supply_ref: RawSupplyRef,
+    }
+
+    public fun initialize(account: &signer, constructor_ref: &ConstructorRef) {
+        assert!(signer::address_of(account) == @0xcafe, 1);
+        let balance_ref = fungible_asset::generate_raw_balance_ref(constructor_ref);
+        let supply_ref = fungible_asset::generate_raw_supply_ref(constructor_ref);
+        move_to<BalanceStore>(account, BalanceStore { balance_ref, supply_ref });
+
+        let balance_value = function_info::new_function_info(
+            account,
+            string::utf8(b"clamped_token"),
+            string::utf8(b"derived_balance"),
+        );
+        let supply_value = function_info::new_function_info(
+            account,
+            string::utf8(b"clamped_token"),
+            string::utf8(b"derived_supply"),
+        );
+
+        let withdraw = function_info::new_function_info(
+            account,
+            string::utf8(b"clamped_token"),
+            string::utf8(b"withdraw"),
+        );
+
+        let deposit = function_info::new_function_info(
+            account,
+            string::utf8(b"clamped_token"),
+            string::utf8(b"deposit"),
+        );
+
+        dispatchable_fungible_asset::register_dispatch_functions(
+            constructor_ref,
+            option::some(withdraw),
+            option::some(deposit),
+            option::some(balance_value)
+        );
+        dispatchable_fungible_asset::register_derive_supply_dispatch_function(
+            constructor_ref,
+            option::some(supply_value)
+        );
+    }
+
+    public fun derived_balance<T: key>(store: Object<T>): u64 acquires BalanceStore {
+        fungible_asset::balance_with_ref(
+            &borrow_global<BalanceStore>(@0xcafe).balance_ref,
+            store
+        )
+    }
+
+    public fun derived_supply<T: key>(metadata: Object<T>): Option<u128> acquires BalanceStore {
+        option::some(option::extract(&mut fungible_asset::supply_with_ref(
+            &borrow_global<BalanceStore>(@0xcafe).supply_ref,
+            metadata
+        )))
+    }
+
+    public fun withdraw<T: key>(
+        store: Object<T>,
+        amount: u64,
+        transfer_ref: &TransferRef,
+    ): FungibleAsset {
+        // Clamp the max amount of asset to withdraw: at most 10 can be withdrawn each call.
+        assert!(amount <= 10, 0);
+        fungible_asset::withdraw_with_ref(transfer_ref, store, amount)
+    }
+
+    public fun deposit<T: key>(
+        store: Object<T>,
+        fa: FungibleAsset,
+        transfer_ref: &TransferRef,
+    ) {
+        fungible_asset::deposit_with_ref(transfer_ref, store, fa)
+    }
+}
diff --git a/aptos-move/framework/aptos-framework/tests/clamped_token_tests.move b/aptos-move/framework/aptos-framework/tests/clamped_token_tests.move
new file mode 100644
index 00000000000..ae6daee5271
--- /dev/null
+++ b/aptos-move/framework/aptos-framework/tests/clamped_token_tests.move
@@ -0,0 +1,55 @@
+#[test_only]
+module aptos_framework::clamped_token_tests {
+    use aptos_framework::fungible_asset::{Self, Metadata, TestToken};
+    use aptos_framework::dispatchable_fungible_asset;
+    use aptos_framework::object;
+    use 0xcafe::clamped_token;
+    use std::option;
+
+    #[test(creator = @0xcafe)]
+    fun test_clamped(
+        creator: &signer,
+    ) {
+        let (creator_ref, token_object) = fungible_asset::create_test_token(creator);
+        let (mint, _, _, _) = fungible_asset::init_test_metadata(&creator_ref);
+        let metadata = object::convert<TestToken, Metadata>(token_object);
+
+        let creator_store = fungible_asset::create_test_store(creator, metadata);
+
+        clamped_token::initialize(creator, &creator_ref);
+
+        assert!(dispatchable_fungible_asset::derived_supply(metadata) == option::some(0), 2);
+        // Mint
+        let fa = fungible_asset::mint(&mint, 100);
+        dispatchable_fungible_asset::deposit(creator_store, fa);
+
+        assert!(dispatchable_fungible_asset::derived_balance(creator_store) == 100, 4);
+        assert!(dispatchable_fungible_asset::derived_supply(metadata) == option::some(100), 5);
+
+        let fa = dispatchable_fungible_asset::withdraw(creator, creator_store, 5);
+        dispatchable_fungible_asset::deposit(creator_store, fa);
+    }
+
+    #[test(creator = @0xcafe)]
+    #[expected_failure(abort_code = 0, location = 0xcafe::clamped_token)]
+    fun test_clamped_aborted(
+        creator: &signer,
+    ) {
+        let (creator_ref, token_object) = fungible_asset::create_test_token(creator);
+        let (mint, _, _, _) = fungible_asset::init_test_metadata(&creator_ref);
+        let metadata = object::convert<TestToken, Metadata>(token_object);
+
+        let creator_store = fungible_asset::create_test_store(creator, metadata);
+
+        clamped_token::initialize(creator, &creator_ref);
+
+        assert!(dispatchable_fungible_asset::derived_supply(metadata) == option::some(0), 2);
+        // Mint
+        let fa = fungible_asset::mint(&mint, 100);
+        dispatchable_fungible_asset::deposit(creator_store, fa);
+
+        // Failed to withdraw as it exceeds the withdraw limit.
+        let fa = dispatchable_fungible_asset::withdraw(creator, creator_store, 20);
+        dispatchable_fungible_asset::deposit(creator_store, fa);
+    }
+}
diff --git a/aptos-move/framework/aptos-framework/tests/nil_op_token_tests.move b/aptos-move/framework/aptos-framework/tests/nil_op_token_tests.move
index d4bf6abbdba..985fab258bc 100644
--- a/aptos-move/framework/aptos-framework/tests/nil_op_token_tests.move
+++ b/aptos-move/framework/aptos-framework/tests/nil_op_token_tests.move
@@ -7,7 +7,6 @@ module aptos_framework::nil_op_token_tests {
     use std::option;
 
     #[test(creator = @0xcafe)]
-    #[expected_failure(abort_code=0x70002, location=aptos_framework::dispatchable_fungible_asset)]
     fun test_nil_op_token(
         creator: &signer,
     ) {
@@ -26,7 +25,6 @@ module aptos_framework::nil_op_token_tests {
         // Deposit will cause an re-entrant call into dispatchable_fungible_asset
         dispatchable_fungible_asset::deposit(creator_store, fa);
 
-        // Withdraw will fail because it's not drawing the basic amount.
         let fa = dispatchable_fungible_asset::withdraw(creator, creator_store, 10);
         dispatchable_fungible_asset::deposit(creator_store, fa);
     }
diff --git a/aptos-move/framework/aptos-framework/tests/ten_x_token.move b/aptos-move/framework/aptos-framework/tests/ten_x_token.move
index a1ac6406265..42799fdfa3e 100644
--- a/aptos-move/framework/aptos-framework/tests/ten_x_token.move
+++ b/aptos-move/framework/aptos-framework/tests/ten_x_token.move
@@ -1,6 +1,6 @@
 #[test_only]
 module 0xcafe::ten_x_token {
-    use aptos_framework::fungible_asset::{Self, RawBalanceRef, RawSupplyRef};
+    use aptos_framework::fungible_asset::{Self, FungibleAsset, RawBalanceRef, RawSupplyRef, TransferRef};
     use aptos_framework::dispatchable_fungible_asset;
     use aptos_framework::object::{ConstructorRef, Object};
     use aptos_framework::function_info;
@@ -31,10 +31,23 @@ module 0xcafe::ten_x_token {
             string::utf8(b"ten_x_token"),
             string::utf8(b"derived_supply"),
         );
+
+        let withdraw = function_info::new_function_info(
+            account,
+            string::utf8(b"ten_x_token"),
+            string::utf8(b"withdraw"),
+        );
+
+        let deposit = function_info::new_function_info(
+            account,
+            string::utf8(b"ten_x_token"),
+            string::utf8(b"deposit"),
+        );
+
         dispatchable_fungible_asset::register_dispatch_functions(
             constructor_ref,
-            option::none(),
-            option::none(),
+            option::some(withdraw),
+            option::some(deposit),
             option::some(balance_value)
         );
         dispatchable_fungible_asset::register_derive_supply_dispatch_function(
@@ -58,4 +71,20 @@ module 0xcafe::ten_x_token {
             metadata
         )) * 10)
     }
+
+    public fun withdraw<T: key>(
+        store: Object<T>,
+        amount: u64,
+        transfer_ref: &TransferRef,
+    ): FungibleAsset {
+        fungible_asset::withdraw_with_ref(transfer_ref, store, amount)
+    }
+
+    public fun deposit<T: key>(
+        store: Object<T>,
+        fa: FungibleAsset,
+        transfer_ref: &TransferRef,
+    ) {
+        fungible_asset::deposit_with_ref(transfer_ref, store, fa)
+    }
 }


Reviews:
user: login: movekevin
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-14T21:50:25Z
user: login: movekevin
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-14T21:50:47Z
user: login: lightmark
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-16T19:52:23Z

Review comments:
diff_hunk: @@ -7,7 +7,6 @@ module aptos_framework::nil_op_token_tests {
     use std::option;
 
     #[test(creator = @0xcafe)]
-    #[expected_failure(abort_code=0x70002, location=aptos_framework::dispatchable_fungible_asset)]
     fun test_nil_op_token(
path: aptos-move/framework/aptos-framework/tests/nil_op_token_tests.move
user: login: movekevin
body: Can you update the test name to record the new expected behavior now?
created_at: 2025-04-14T21:50:25Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,89 @@
+#[test_only]
path: aptos-move/framework/aptos-framework/tests/clamped_token.move
user: login: lightmark
body: what does clamped mean here? 
add some comments?
created_at: 2025-04-16T19:51:29Z
author_association: CONTRIBUTOR

PR comments:
user: login: trunk-io[bot]
created_at: 2025-04-09T17:04:50Z
body: <!-- Trunk Metrics Report -->



<details>
<summary>
⏱️ 41m total CI duration on this PR
</summary>

| Job | Cumulative Duration | Recent Runs |
|---|---|---|
| [rust-targeted-unit-tests](https://github.com/aptos-labs/aptos-core/actions/runs/14363408217/job/40270388471) | 19m | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14363408217/job/40270388471)&#8201; |
| [check-dynamic-deps](https://github.com/aptos-labs/aptos-core/actions/runs/14479702969/job/40613815334) | 7m | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14362920681/job/40268751334)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14363408190/job/40270364369)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14479702969/job/40613815334)&#8201; |
| [rust-cargo-deny](https://github.com/aptos-labs/aptos-core/actions/runs/14479702929/job/40613831326) | 5m | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14362920700/job/40268775037)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14363408217/job/40270385582)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14479702929/job/40613831326)&#8201; |
| [rust-lints](https://github.com/aptos-labs/aptos-core/actions/runs/14363408217/job/40270385036) | 4m | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14363408217/job/40270385036)&#8201; |
| [rust-check-merge-base](https://github.com/aptos-labs/aptos-core/actions/runs/14363408217/job/40270386622) | 3m | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14363408217/job/40270386622)&#8201; |
| [general-lints](https://github.com/aptos-labs/aptos-core/actions/runs/14479702929/job/40613831679) | 1m | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14362920700/job/40268775667)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14363408217/job/40270386181)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14479702929/job/40613831679)&#8201; |
| [semgrep/ci](https://github.com/aptos-labs/aptos-core/actions/runs/14479702967/job/40613815162) | 1m | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14362920669/job/40268750643)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14363408139/job/40270363850)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14479702967/job/40613815162)&#8201; |
| [file_change_determinator](https://github.com/aptos-labs/aptos-core/actions/runs/14479702929/job/40613816116) | 35s | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14362920700/job/40268753228)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14363408217/job/40270365300)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14479702929/job/40613816116)&#8201; |
| [permission-check](https://github.com/aptos-labs/aptos-core/actions/runs/14479702747/job/40613814659) | 6s | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14362920670/job/40268750721)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14479702747/job/40613814659)&#8201; |
| [permission-check](https://github.com/aptos-labs/aptos-core/actions/runs/14479702732/job/40613814842) | 5s | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14362920664/job/40268750719)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14479702732/job/40613814842)&#8201; |
</details>

<sub>[settings](https://app.trunk.io/aptoslabs/settings/repo?repo=aptos-labs/aptos-core&utm_source=gh-comment#ci-analytics) ⋅ [feedback](https://form.typeform.com/to/w7Iv34u2?utm_source=gh-comment) ⋅ [docs](https://docs.trunk.io/ci-analytics/pull-request-metrics?utm_source=gh-comment) ⋅ learn more about [trunk.io](https://trunk.io?utm_source=gh-comment)</sub>
author_association: NONE
user: login: runtian-zhou
created_at: 2025-04-09T17:05:02Z
body: * **#16313** <a href="https://app.graphite.dev/github/pr/aptos-labs/aptos-core/16313?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a> 👈 <a href="https://app.graphite.dev/github/pr/aptos-labs/aptos-core/16313?utm_source=stack-comment-view-in-graphite" target="_blank">(View in Graphite)</a>
* `main`


<h2></h2>

This stack of pull requests is managed by <a href="https://graphite.dev?utm-source=stack-comment"><b>Graphite</b></a>. Learn more about <a href="https://stacking.dev/?utm_source=stack-comment">stacking</a>.
<!-- Current dependencies on/for this PR: -->
author_association: CONTRIBUTOR
user: login: runtian-zhou
created_at: 2025-04-12T00:10:58Z
body: Drafted AIP for discussion and justification for this change: https://github.com/aptos-foundation/AIPs/pull/582
author_association: CONTRIBUTOR
user: login: movekevin
created_at: 2025-04-14T21:51:17Z
body: Can you add a test where the override function clamps down the amount to max amount users can withdraw?
author_association: CONTRIBUTOR
user: login: runtian-zhou
created_at: 2025-04-15T21:26:25Z
body: Test added
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-17T05:51:38Z
body: ### Forge is running suite `framework_upgrade` on `463dda939a7bdc1266946e824a1eb309f300eaf8` ==> `12b0f381ca1218ddf51177ccd77ad5ab630a1a77`
* [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-framework-upgrade-pr-16313&var-metrics_source=All&var-chain_name=forge-0&refresh=10s&from=now-15m&to=now)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-framework-upgrade-pr-16313%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-framework-upgrade-pr-16313%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16313%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16313%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22quickRange%22%3A%20%2230m%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16313&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16313&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14508828320)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-framework-upgrade -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-17T05:51:41Z
body: ### Forge is running suite `compat` on `463dda939a7bdc1266946e824a1eb309f300eaf8` ==> `12b0f381ca1218ddf51177ccd77ad5ab630a1a77`
* [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-compat-pr-16313&var-metrics_source=All&var-chain_name=forge-0&refresh=10s&from=now-15m&to=now)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-compat-pr-16313%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-compat-pr-16313%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16313%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16313%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22quickRange%22%3A%20%2230m%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16313&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16313&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14508828320)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-compat -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-17T05:51:42Z
body: ### Forge is running suite `realistic_env_max_load` on `12b0f381ca1218ddf51177ccd77ad5ab630a1a77`
* [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-e2e-pr-16313&var-metrics_source=All&var-chain_name=forge-1&refresh=10s&from=now-15m&to=now)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-e2e-pr-16313%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-e2e-pr-16313%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16313%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16313%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22quickRange%22%3A%20%2230m%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16313&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16313&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14508828320)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-e2e -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-17T06:02:56Z
body: ### :x: Forge suite `compat` failure on `463dda939a7bdc1266946e824a1eb309f300eaf8` ==> `12b0f381ca1218ddf51177ccd77ad5ab630a1a77`
```
Compatibility test results for 463dda939a7bdc1266946e824a1eb309f300eaf8 ==> 12b0f381ca1218ddf51177ccd77ad5ab630a1a77 (PR)
1. Check liveness of validators at old version: 463dda939a7bdc1266946e824a1eb309f300eaf8
compatibility::simple-validator-upgrade::liveness-check : committed: 7308.85 txn/s, latency: 3827.89 ms, (p50: 3500 ms, p70: 3800, p90: 4500 ms, p99: 18200 ms), latency samples: 285280
2. Upgrading first Validator to new version: 12b0f381ca1218ddf51177ccd77ad5ab630a1a77
compatibility::simple-validator-upgrade::single-validator-upgrade : committed: 2766.98 txn/s, latency: 12010.01 ms, (p50: 13700 ms, p70: 13900, p90: 14200 ms, p99: 14400 ms), latency samples: 103380
3. Upgrading rest of first batch to new version: 12b0f381ca1218ddf51177ccd77ad5ab630a1a77
Test Failed: Tried executing 10 txns, request counters: "success 1, failed submit [0], failed wait [9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9], by client: [(1, 0, 162): http://aptos-node-0-validator.forge-compat-pr-16313.svc:8080/v1/]"

Caused by:
    Unknown error Transaction expired, without being seen in mempool. It is guaranteed it will not be committed on chain.

Stack backtrace:
   0: anyhow::error::<impl core::convert::From<E> for anyhow::Error>::from
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/anyhow-1.0.89/src/error.rs:564:25
   1: <core::result::Result<T,F> as core::ops::try_trait::FromResidual<core::result::Result<core::convert::Infallible,E>>>::from_residual
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/result.rs:1964:27
   2: aptos_transaction_emitter_lib::emitter::transaction_executor::RestApiReliableTransactionSubmitter::submit_check_and_retry::{{closure}}
             at ./crates/transaction-emitter-lib/src/emitter/transaction_executor.rs:142:28
   3: <futures_util::future::maybe_done::MaybeDone<Fut> as core::future::future::Future>::poll
             at /usr/local/cargo/git/checkouts/futures-rs-2e928889f587a511/1845d7d/futures-util/src/future/maybe_done.rs:95:38
   4: <futures_util::future::join_all::JoinAll<F> as core::future::future::Future>::poll
             at /usr/local/cargo/git/checkouts/futures-rs-2e928889f587a511/1845d7d/futures-util/src/future/join_all.rs:143:24
   5: <aptos_transaction_emitter_lib::emitter::transaction_executor::RestApiReliableTransactionSubmitter as aptos_transaction_generator_lib::ReliableTransactionSubmitter>::execute_transactions_with_counter::{{closure}}
             at ./crates/transaction-emitter-lib/src/emitter/transaction_executor.rs:344:10
   6: <core::pin::Pin<P> as core::future::future::Future>::poll
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/future/future.rs:123:9
   7: aptos_transaction_emitter_lib::emitter::account_minter::AccountMinter::create_and_fund_seed_accounts::{{closure}}
             at ./crates/transaction-emitter-lib/src/emitter/account_minter.rs:396:18
   8: aptos_transaction_emitter_lib::emitter::account_minter::AccountMinter::create_and_fund_accounts::{{closure}}
             at ./crates/transaction-emitter-lib/src/emitter/account_minter.rs:298:10
   9: aptos_transaction_emitter_lib::emitter::account_minter::bulk_create_accounts::{{closure}}
             at ./crates/transaction-emitter-lib/src/emitter/account_minter.rs:686:14
  10: aptos_transaction_emitter_lib::emitter::TxnEmitter::start_job::{{closure}}
             at ./crates/transaction-emitter-lib/src/emitter/mod.rs:812:10
  11: aptos_transaction_emitter_lib::emitter::TxnEmitter::emit_txn_for_impl::{{closure}}
             at ./crates/transaction-emitter-lib/src/emitter/mod.rs:926:14
  12: aptos_transaction_emitter_lib::emitter::TxnEmitter::emit_txn_for::{{closure}}
             at ./crates/transaction-emitter-lib/src/emitter/mod.rs:958:14
  13: aptos_testcases::generate_traffic::{{closure}}
             at ./testsuite/testcases/src/lib.rs:185:10
  14: <aptos_testcases::compatibility_test::SimpleValidatorUpgrade as aptos_forge::interface::network::NetworkTest>::run::{{closure}}
             at ./testsuite/testcases/src/compatibility_test.rs:155:79
  15: <core::pin::Pin<P> as core::future::future::Future>::poll
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/future/future.rs:123:9
  16: tokio::runtime::park::CachedParkThread::block_on::{{closure}}
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/park.rs:281:63
  17: tokio::runtime::coop::with_budget
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/coop.rs:107:5
  18: tokio::runtime::coop::budget
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/coop.rs:73:5
  19: tokio::runtime::park::CachedParkThread::block_on
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/park.rs:281:31
  20: tokio::runtime::context::blocking::BlockingRegionGuard::block_on
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/context/blocking.rs:66:9
  21: tokio::runtime::handle::Handle::block_on_inner::{{closure}}
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/handle.rs:324:22
  22: tokio::runtime::context::runtime::enter_runtime
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/context/runtime.rs:65:16
  23: tokio::runtime::handle::Handle::block_on_inner
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/handle.rs:323:9
  24: tokio::runtime::handle::Handle::block_on
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/handle.rs:302:18
  25: aptos_forge::runner::Forge<F>::run
             at ./testsuite/forge/src/runner.rs:332:50
  26: forge::run_forge_with_changelog
             at ./testsuite/forge-cli/src/main.rs:430:24
  27: forge::main
             at ./testsuite/forge-cli/src/main.rs:333:21
  28: core::ops::function::FnOnce::call_once
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/ops/function.rs:250:5
  29: std::sys_common::backtrace::__rust_begin_short_backtrace
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/sys_common/backtrace.rs:155:18
  30: std::rt::lang_start::{{closure}}
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/rt.rs:166:18
  31: core::ops::function::impls::<impl core::ops::function::FnOnce<A> for &F>::call_once
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/ops/function.rs:284:13
  32: std::panicking::try::do_call
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:552:40
  33: std::panicking::try
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:516:19
  34: std::panic::catch_unwind
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panic.rs:146:14
  35: std::rt::lang_start_internal::{{closure}}
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/rt.rs:148:48
  36: std::panicking::try::do_call
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:552:40
  37: std::panicking::try
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:516:19
  38: std::panic::catch_unwind
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panic.rs:146:14
  39: std::rt::lang_start_internal
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/rt.rs:148:20
  40: main
  41: __libc_start_main
  42: _start
Trailing Log Lines:
  37: std::panicking::try
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:516:19
  38: std::panic::catch_unwind
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panic.rs:146:14
  39: std::rt::lang_start_internal
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/rt.rs:148:20
  40: main
  41: __libc_start_main
  42: _start

=== BEGIN JUNIT ===
<?xml version="1.0" encoding="UTF-8"?>
<testsuites name="forge" tests="1" failures="1" errors="0" uuid="91f02f39-7a86-424e-b60f-3598b6f9f3dd">
    <testsuite name="compat" tests="1" disabled="0" errors="0" failures="1">
        <testcase name="compatibility::simple-validator-upgrade">
            <failure message="Tried executing 10 txns, request counters: &quot;success 1, failed submit [0], failed wait [9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9], by client: [(1, 0, 162): http://aptos-node-0-validator.forge-compat-pr-16313.svc:8080/v1/]&quot;

Caused by:
    Unknown error Transaction expired, without being seen in mempool. It is guaranteed it will not be committed on chain.

Stack backtrace:
   0: anyhow::error::&lt;impl core::convert::From&lt;E&gt; for anyhow::Error&gt;::from
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/anyhow-1.0.89/src/error.rs:564:25
   1: &lt;core::result::Result&lt;T,F&gt; as core::ops::try_trait::FromResidual&lt;core::result::Result&lt;core::convert::Infallible,E&gt;&gt;&gt;::from_residual
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/result.rs:1964:27
   2: aptos_transaction_emitter_lib::emitter::transaction_executor::RestApiReliableTransactionSubmitter::submit_check_and_retry::{{closure}}
             at ./crates/transaction-emitter-lib/src/emitter/transaction_executor.rs:142:28
   3: &lt;futures_util::future::maybe_done::MaybeDone&lt;Fut&gt; as core::future::future::Future&gt;::poll
             at /usr/local/cargo/git/checkouts/futures-rs-2e928889f587a511/1845d7d/futures-util/src/future/maybe_done.rs:95:38
   4: &lt;futures_util::future::join_all::JoinAll&lt;F&gt; as core::future::future::Future&gt;::poll
             at /usr/local/cargo/git/checkouts/futures-rs-2e928889f587a511/1845d7d/futures-util/src/future/join_all.rs:143:24
   5: &lt;aptos_transaction_emitter_lib::emitter::transaction_executor::RestApiReliableTransactionSubmitter as aptos_transaction_generator_lib::ReliableTransactionSubmitter&gt;::execute_transactions_with_counter::{{closure}}
             at ./crates/transaction-emitter-lib/src/emitter/transaction_executor.rs:344:10
   6: &lt;core::pin::Pin&lt;P&gt; as core::future::future::Future&gt;::poll
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/future/future.rs:123:9
   7: aptos_transaction_emitter_lib::emitter::account_minter::AccountMinter::create_and_fund_seed_accounts::{{closure}}
             at ./crates/transaction-emitter-lib/src/emitter/account_minter.rs:396:18
   8: aptos_transaction_emitter_lib::emitter::account_minter::AccountMinter::create_and_fund_accounts::{{closure}}
             at ./crates/transaction-emitter-lib/src/emitter/account_minter.rs:298:10
   9: aptos_transaction_emitter_lib::emitter::account_minter::bulk_create_accounts::{{closure}}
             at ./crates/transaction-emitter-lib/src/emitter/account_minter.rs:686:14
  10: aptos_transaction_emitter_lib::emitter::TxnEmitter::start_job::{{closure}}
             at ./crates/transaction-emitter-lib/src/emitter/mod.rs:812:10
  11: aptos_transaction_emitter_lib::emitter::TxnEmitter::emit_txn_for_impl::{{closure}}
             at ./crates/transaction-emitter-lib/src/emitter/mod.rs:926:14
  12: aptos_transaction_emitter_lib::emitter::TxnEmitter::emit_txn_for::{{closure}}
             at ./crates/transaction-emitter-lib/src/emitter/mod.rs:958:14
  13: aptos_testcases::generate_traffic::{{closure}}
             at ./testsuite/testcases/src/lib.rs:185:10
  14: &lt;aptos_testcases::compatibility_test::SimpleValidatorUpgrade as aptos_forge::interface::network::NetworkTest&gt;::run::{{closure}}
             at ./testsuite/testcases/src/compatibility_test.rs:155:79
  15: &lt;core::pin::Pin&lt;P&gt; as core::future::future::Future&gt;::poll
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/future/future.rs:123:9
  16: tokio::runtime::park::CachedParkThread::block_on::{{closure}}
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/park.rs:281:63
  17: tokio::runtime::coop::with_budget
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/coop.rs:107:5
  18: tokio::runtime::coop::budget
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/coop.rs:73:5
  19: tokio::runtime::park::CachedParkThread::block_on
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/park.rs:281:31
  20: tokio::runtime::context::blocking::BlockingRegionGuard::block_on
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/context/blocking.rs:66:9
  21: tokio::runtime::handle::Handle::block_on_inner::{{closure}}
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/handle.rs:324:22
  22: tokio::runtime::context::runtime::enter_runtime
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/context/runtime.rs:65:16
  23: tokio::runtime::handle::Handle::block_on_inner
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/handle.rs:323:9
  24: tokio::runtime::handle::Handle::block_on
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/handle.rs:302:18
  25: aptos_forge::runner::Forge&lt;F&gt;::run
             at ./testsuite/forge/src/runner.rs:332:50
  26: forge::run_forge_with_changelog
             at ./testsuite/forge-cli/src/main.rs:430:24
  27: forge::main
             at ./testsuite/forge-cli/src/main.rs:333:21
  28: core::ops::function::FnOnce::call_once
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/ops/function.rs:250:5
  29: std::sys_common::backtrace::__rust_begin_short_backtrace
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/sys_common/backtrace.rs:155:18
  30: std::rt::lang_start::{{closure}}
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/rt.rs:166:18
  31: core::ops::function::impls::&lt;impl core::ops::function::FnOnce&lt;A&gt; for &amp;F&gt;::call_once
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/ops/function.rs:284:13
  32: std::panicking::try::do_call
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:552:40
  33: std::panicking::try
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:516:19
  34: std::panic::catch_unwind
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panic.rs:146:14
  35: std::rt::lang_start_internal::{{closure}}
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/rt.rs:148:48
  36: std::panicking::try::do_call
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:552:40
  37: std::panicking::try
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:516:19
  38: std::panic::catch_unwind
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panic.rs:146:14
  39: std::rt::lang_start_internal
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/rt.rs:148:20
  40: main
  41: __libc_start_main
  42: _start"/>
        </testcase>
    </testsuite>
</testsuites>
=== END JUNIT ===

Swarm logs can be found here: See fgi output for more information.
[2025-04-17T06:02:51Z INFO  aptos_forge::backend::k8s::cluster_helper] Deleting namespace forge-compat-pr-16313: Some(NamespaceStatus { conditions: None, phase: Some("Terminating") })
[2025-04-17T06:02:51Z INFO  aptos_forge::backend::k8s::cluster_helper] aptos-node resources for Forge removed in namespace: forge-compat-pr-16313

failures:
    compatibility::simple-validator-upgrade
Failed to run tests:
Tests Failed

test result: FAILED. 0 passed; 1 failed; 0 filtered out

Error: Tests Failed

Stack backtrace:
   0: anyhow::error::<impl anyhow::Error>::msg
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/anyhow-1.0.89/src/error.rs:85:36
   1: aptos_forge::runner::Forge<F>::run
             at ./testsuite/forge/src/runner.rs:358:13
   2: forge::run_forge_with_changelog
             at ./testsuite/forge-cli/src/main.rs:430:24
   3: forge::main
             at ./testsuite/forge-cli/src/main.rs:333:21
   4: core::ops::function::FnOnce::call_once
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/ops/function.rs:250:5
   5: std::sys_common::backtrace::__rust_begin_short_backtrace
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/sys_common/backtrace.rs:155:18
   6: std::rt::lang_start::{{closure}}
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/rt.rs:166:18
   7: core::ops::function::impls::<impl core::ops::function::FnOnce<A> for &F>::call_once
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/ops/function.rs:284:13
   8: std::panicking::try::do_call
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:552:40
   9: std::panicking::try
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:516:19
  10: std::panic::catch_unwind
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panic.rs:146:14
  11: std::rt::lang_start_internal::{{closure}}
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/rt.rs:148:48
  12: std::panicking::try::do_call
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:552:40
  13: std::panicking::try
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:516:19
  14: std::panic::catch_unwind
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panic.rs:146:14
  15: std::rt::lang_start_internal
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/rt.rs:148:20
  16: main
  17: __libc_start_main
  18: _start
Debugging output:
NAME                                   READY   STATUS      RESTARTS   AGE
aptos-node-0-validator-0               1/1     Running     0          4m24s
aptos-node-1-validator-0               1/1     Running     0          6m48s
aptos-node-2-validator-0               1/1     Running     0          10m
aptos-node-3-validator-0               1/1     Running     0          10m
forge-testnet-deployer-kmqx4           0/1     Completed   0          10m
genesis-aptos-genesis-eforge20-n4pm5   0/1     Completed   0          10m

```
* [Grafana dashboard](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-compat-pr-16313&var-metrics_source=All&var-chain_name=forge-0&from=1744869108000&to=1744869774000)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-compat-pr-16313%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-compat-pr-16313%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1744869108000&end=1744869774000)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16313%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16313%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22startTime%22%3A%20%222025-04-17T05%3A51%3A48.771692%2B00%3A00%22%2C%20%22endTime%22%3A%20%222025-04-17T06%3A02%3A54.592702%2B00%3A00%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1744869108&until=1744869774&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16313&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1744869108&until=1744869774&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16313&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14508828320)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-compat -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-17T06:04:47Z
body: ### :white_check_mark: Forge suite `framework_upgrade` success on `463dda939a7bdc1266946e824a1eb309f300eaf8` ==> `12b0f381ca1218ddf51177ccd77ad5ab630a1a77`
```
Compatibility test results for 463dda939a7bdc1266946e824a1eb309f300eaf8 ==> 12b0f381ca1218ddf51177ccd77ad5ab630a1a77 (PR)
Upgrade the nodes to version: 12b0f381ca1218ddf51177ccd77ad5ab630a1a77
framework_upgrade::framework-upgrade::full-framework-upgrade : committed: 1731.48 txn/s, submitted: 1737.84 txn/s, failed submission: 6.36 txn/s, expired: 6.36 txn/s, latency: 1726.55 ms, (p50: 1600 ms, p70: 1900, p90: 2400 ms, p99: 3400 ms), latency samples: 152420
framework_upgrade::framework-upgrade::full-framework-upgrade : committed: 1653.69 txn/s, submitted: 1659.88 txn/s, failed submission: 6.19 txn/s, expired: 6.19 txn/s, latency: 1787.39 ms, (p50: 1600 ms, p70: 1900, p90: 2700 ms, p99: 3700 ms), latency samples: 149580
5. check swarm health
Compatibility test for 463dda939a7bdc1266946e824a1eb309f300eaf8 ==> 12b0f381ca1218ddf51177ccd77ad5ab630a1a77 passed
Upgrade the remaining nodes to version: 12b0f381ca1218ddf51177ccd77ad5ab630a1a77
framework_upgrade::framework-upgrade::full-framework-upgrade : committed: 1680.45 txn/s, submitted: 1684.25 txn/s, failed submission: 3.79 txn/s, expired: 3.79 txn/s, latency: 1832.61 ms, (p50: 1600 ms, p70: 1800, p90: 2700 ms, p99: 7000 ms), latency samples: 150641
Test Ok
```
* [Grafana dashboard](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-framework-upgrade-pr-16313&var-metrics_source=All&var-chain_name=forge-0&from=1744869104000&to=1744869884000)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-framework-upgrade-pr-16313%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-framework-upgrade-pr-16313%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1744869104000&end=1744869884000)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16313%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16313%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22startTime%22%3A%20%222025-04-17T05%3A51%3A44.674444%2B00%3A00%22%2C%20%22endTime%22%3A%20%222025-04-17T06%3A04%3A44.779592%2B00%3A00%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1744869104&until=1744869884&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16313&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1744869104&until=1744869884&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16313&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14508828320)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-framework-upgrade -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-17T06:22:55Z
body: ### :x: Forge suite `realistic_env_max_load` failure on `12b0f381ca1218ddf51177ccd77ad5ab630a1a77`
```
two traffics test: inner traffic : committed: 12373.13 txn/s, latency: 3210.41 ms, (p50: 2700 ms, p70: 3000, p90: 3600 ms, p99: 11500 ms), latency samples: 4704520
two traffics test : committed: 99.47 txn/s, submitted: 99.97 txn/s, failed submission: 0.50 txn/s, expired: 0.50 txn/s, latency: 4068.52 ms, (p50: 2600 ms, p70: 3000, p90: 8500 ms, p99: 28000 ms), latency samples: 1820
Latency breakdown for phase 0: ["MempoolToBlockCreation: max: 1.127, avg: 0.496", "ConsensusProposalToOrdered: max: 0.303, avg: 0.299", "ConsensusOrderedToCommit: max: 0.590, avg: 0.442", "ConsensusProposalToCommit: max: 0.893, avg: 0.740"]
Test Failed: check for success

Caused by:
    0: Failed ensuring no fullnode restarted
    1: Container fullnode in pod aptos-node-1-fullnode-eforge244-0 restarted 1 times 

Stack backtrace:
   0: anyhow::error::<impl anyhow::Error>::msg
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/anyhow-1.0.89/src/error.rs:85:36
   1: aptos_forge::backend::k8s::stateful_set::check_for_container_restart::{{closure}}::{{closure}}::{{closure}}
             at ./testsuite/forge/src/backend/k8s/stateful_set.rs:294:29
   2: <core::pin::Pin<P> as core::future::future::Future>::poll
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/future/future.rs:123:9
   3: aptos_retrier::retry_async::{{closure}}
             at ./crates/aptos-retrier/src/lib.rs:46:27
   4: aptos_forge::backend::k8s::stateful_set::check_for_container_restart::{{closure}}
             at ./testsuite/forge/src/backend/k8s/stateful_set.rs:311:6
   5: <aptos_forge::backend::k8s::swarm::K8sSwarm as aptos_forge::interface::swarm::Swarm>::ensure_no_fullnode_restart::{{closure}}
             at ./testsuite/forge/src/backend/k8s/swarm.rs:394:14
   6: <core::pin::Pin<P> as core::future::future::Future>::poll
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/future/future.rs:123:9
   7: aptos_forge::success_criteria::SuccessCriteriaChecker::check_for_success::{{closure}}
             at ./testsuite/forge/src/success_criteria.rs:336:18
   8: aptos_forge::interface::network::NetworkContext::check_for_success::{{closure}}
             at ./testsuite/forge/src/interface/network.rs:112:10
   9: <dyn aptos_testcases::NetworkLoadTest as aptos_forge::interface::network::NetworkTest>::run::{{closure}}
             at ./testsuite/testcases/src/lib.rs:333:14
  10: <core::pin::Pin<P> as core::future::future::Future>::poll
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/future/future.rs:123:9
  11: <aptos_testcases::two_traffics_test::TwoTrafficsTest as aptos_forge::interface::network::NetworkTest>::run::{{closure}}
             at ./testsuite/testcases/src/two_traffics_test.rs:77:47
  12: <core::pin::Pin<P> as core::future::future::Future>::poll
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/future/future.rs:123:9
  13: <aptos_testcases::CompositeNetworkTest as aptos_forge::interface::network::NetworkTest>::run::{{closure}}
             at ./testsuite/testcases/src/lib.rs:639:37
  14: <core::pin::Pin<P> as core::future::future::Future>::poll
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/future/future.rs:123:9
  15: tokio::runtime::park::CachedParkThread::block_on::{{closure}}
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/park.rs:281:63
  16: tokio::runtime::coop::with_budget
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/coop.rs:107:5
  17: tokio::runtime::coop::budget
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/coop.rs:73:5
  18: tokio::runtime::park::CachedParkThread::block_on
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/park.rs:281:31
  19: tokio::runtime::context::blocking::BlockingRegionGuard::block_on
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/context/blocking.rs:66:9
  20: tokio::runtime::handle::Handle::block_on_inner::{{closure}}
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/handle.rs:324:22
  21: tokio::runtime::context::runtime::enter_runtime
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/context/runtime.rs:65:16
  22: tokio::runtime::handle::Handle::block_on_inner
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/handle.rs:323:9
  23: tokio::runtime::handle::Handle::block_on
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/handle.rs:302:18
  24: aptos_forge::runner::Forge<F>::run
             at ./testsuite/forge/src/runner.rs:332:50
  25: forge::run_forge_with_changelog
             at ./testsuite/forge-cli/src/main.rs:430:24
  26: forge::main
             at ./testsuite/forge-cli/src/main.rs:333:21
  27: core::ops::function::FnOnce::call_once
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/ops/function.rs:250:5
  28: std::sys_common::backtrace::__rust_begin_short_backtrace
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/sys_common/backtrace.rs:155:18
  29: std::rt::lang_start::{{closure}}
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/rt.rs:166:18
  30: core::ops::function::impls::<impl core::ops::function::FnOnce<A> for &F>::call_once
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/ops/function.rs:284:13
  31: std::panicking::try::do_call
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:552:40
  32: std::panicking::try
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:516:19
  33: std::panic::catch_unwind
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panic.rs:146:14
  34: std::rt::lang_start_internal::{{closure}}
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/rt.rs:148:48
  35: std::panicking::try::do_call
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:552:40
  36: std::panicking::try
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:516:19
  37: std::panic::catch_unwind
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panic.rs:146:14
  38: std::rt::lang_start_internal
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/rt.rs:148:20
  39: main
  40: __libc_start_main
  41: _start
Trailing Log Lines:
  36: std::panicking::try
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:516:19
  37: std::panic::catch_unwind
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panic.rs:146:14
  38: std::rt::lang_start_internal
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/rt.rs:148:20
  39: main
  40: __libc_start_main
  41: _start

=== BEGIN JUNIT ===
<?xml version="1.0" encoding="UTF-8"?>
<testsuites name="forge" tests="1" failures="1" errors="0" uuid="e2ef6c76-87c8-41ac-8cd1-8cc8471bed85">
    <testsuite name="local" tests="1" disabled="0" errors="0" failures="1">
        <testcase name="CompositeNetworkTest(network:multi-region-network-emulation(two traffics test)) with ">
            <failure message="check for success

Caused by:
    0: Failed ensuring no fullnode restarted
    1: Container fullnode in pod aptos-node-1-fullnode-eforge244-0 restarted 1 times 

Stack backtrace:
   0: anyhow::error::&lt;impl anyhow::Error&gt;::msg
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/anyhow-1.0.89/src/error.rs:85:36
   1: aptos_forge::backend::k8s::stateful_set::check_for_container_restart::{{closure}}::{{closure}}::{{closure}}
             at ./testsuite/forge/src/backend/k8s/stateful_set.rs:294:29
   2: &lt;core::pin::Pin&lt;P&gt; as core::future::future::Future&gt;::poll
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/future/future.rs:123:9
   3: aptos_retrier::retry_async::{{closure}}
             at ./crates/aptos-retrier/src/lib.rs:46:27
   4: aptos_forge::backend::k8s::stateful_set::check_for_container_restart::{{closure}}
             at ./testsuite/forge/src/backend/k8s/stateful_set.rs:311:6
   5: &lt;aptos_forge::backend::k8s::swarm::K8sSwarm as aptos_forge::interface::swarm::Swarm&gt;::ensure_no_fullnode_restart::{{closure}}
             at ./testsuite/forge/src/backend/k8s/swarm.rs:394:14
   6: &lt;core::pin::Pin&lt;P&gt; as core::future::future::Future&gt;::poll
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/future/future.rs:123:9
   7: aptos_forge::success_criteria::SuccessCriteriaChecker::check_for_success::{{closure}}
             at ./testsuite/forge/src/success_criteria.rs:336:18
   8: aptos_forge::interface::network::NetworkContext::check_for_success::{{closure}}
             at ./testsuite/forge/src/interface/network.rs:112:10
   9: &lt;dyn aptos_testcases::NetworkLoadTest as aptos_forge::interface::network::NetworkTest&gt;::run::{{closure}}
             at ./testsuite/testcases/src/lib.rs:333:14
  10: &lt;core::pin::Pin&lt;P&gt; as core::future::future::Future&gt;::poll
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/future/future.rs:123:9
  11: &lt;aptos_testcases::two_traffics_test::TwoTrafficsTest as aptos_forge::interface::network::NetworkTest&gt;::run::{{closure}}
             at ./testsuite/testcases/src/two_traffics_test.rs:77:47
  12: &lt;core::pin::Pin&lt;P&gt; as core::future::future::Future&gt;::poll
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/future/future.rs:123:9
  13: &lt;aptos_testcases::CompositeNetworkTest as aptos_forge::interface::network::NetworkTest&gt;::run::{{closure}}
             at ./testsuite/testcases/src/lib.rs:639:37
  14: &lt;core::pin::Pin&lt;P&gt; as core::future::future::Future&gt;::poll
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/future/future.rs:123:9
  15: tokio::runtime::park::CachedParkThread::block_on::{{closure}}
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/park.rs:281:63
  16: tokio::runtime::coop::with_budget
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/coop.rs:107:5
  17: tokio::runtime::coop::budget
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/coop.rs:73:5
  18: tokio::runtime::park::CachedParkThread::block_on
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/park.rs:281:31
  19: tokio::runtime::context::blocking::BlockingRegionGuard::block_on
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/context/blocking.rs:66:9
  20: tokio::runtime::handle::Handle::block_on_inner::{{closure}}
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/handle.rs:324:22
  21: tokio::runtime::context::runtime::enter_runtime
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/context/runtime.rs:65:16
  22: tokio::runtime::handle::Handle::block_on_inner
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/handle.rs:323:9
  23: tokio::runtime::handle::Handle::block_on
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/handle.rs:302:18
  24: aptos_forge::runner::Forge&lt;F&gt;::run
             at ./testsuite/forge/src/runner.rs:332:50
  25: forge::run_forge_with_changelog
             at ./testsuite/forge-cli/src/main.rs:430:24
  26: forge::main
             at ./testsuite/forge-cli/src/main.rs:333:21
  27: core::ops::function::FnOnce::call_once
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/ops/function.rs:250:5
  28: std::sys_common::backtrace::__rust_begin_short_backtrace
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/sys_common/backtrace.rs:155:18
  29: std::rt::lang_start::{{closure}}
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/rt.rs:166:18
  30: core::ops::function::impls::&lt;impl core::ops::function::FnOnce&lt;A&gt; for &amp;F&gt;::call_once
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/ops/function.rs:284:13
  31: std::panicking::try::do_call
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:552:40
  32: std::panicking::try
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:516:19
  33: std::panic::catch_unwind
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panic.rs:146:14
  34: std::rt::lang_start_internal::{{closure}}
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/rt.rs:148:48
  35: std::panicking::try::do_call
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:552:40
  36: std::panicking::try
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:516:19
  37: std::panic::catch_unwind
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panic.rs:146:14
  38: std::rt::lang_start_internal
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/rt.rs:148:20
  39: main
  40: __libc_start_main
  41: _start"/>
        </testcase>
    </testsuite>
</testsuites>
=== END JUNIT ===

Swarm logs can be found here: See fgi output for more information.
[2025-04-17T06:22:50Z INFO  aptos_forge::backend::k8s::cluster_helper] Deleting namespace forge-e2e-pr-16313: Some(NamespaceStatus { conditions: None, phase: Some("Terminating") })
[2025-04-17T06:22:50Z INFO  aptos_forge::backend::k8s::cluster_helper] aptos-node resources for Forge removed in namespace: forge-e2e-pr-16313

failures:
    CompositeNetworkTest

test result: FAILED. 0 passed; 1 failed; 0 filtered out

Failed to run tests:
Tests Failed
Error: Tests Failed

Stack backtrace:
   0: anyhow::error::<impl anyhow::Error>::msg
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/anyhow-1.0.89/src/error.rs:85:36
   1: aptos_forge::runner::Forge<F>::run
             at ./testsuite/forge/src/runner.rs:358:13
   2: forge::run_forge_with_changelog
             at ./testsuite/forge-cli/src/main.rs:430:24
   3: forge::main
             at ./testsuite/forge-cli/src/main.rs:333:21
   4: core::ops::function::FnOnce::call_once
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/ops/function.rs:250:5
   5: std::sys_common::backtrace::__rust_begin_short_backtrace
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/sys_common/backtrace.rs:155:18
   6: std::rt::lang_start::{{closure}}
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/rt.rs:166:18
   7: core::ops::function::impls::<impl core::ops::function::FnOnce<A> for &F>::call_once
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/ops/function.rs:284:13
   8: std::panicking::try::do_call
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:552:40
   9: std::panicking::try
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:516:19
  10: std::panic::catch_unwind
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panic.rs:146:14
  11: std::rt::lang_start_internal::{{closure}}
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/rt.rs:148:48
  12: std::panicking::try::do_call
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:552:40
  13: std::panicking::try
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:516:19
  14: std::panic::catch_unwind
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panic.rs:146:14
  15: std::rt::lang_start_internal
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/rt.rs:148:20
  16: main
  17: __libc_start_main
  18: _start
Debugging output:
NAME                                    READY   STATUS      RESTARTS      AGE
aptos-node-0-fullnode-eforge244-0       1/1     Running     0             30m
aptos-node-0-validator-0                1/1     Running     0             30m
aptos-node-1-fullnode-eforge244-0       1/1     Running     1 (22m ago)   30m
aptos-node-1-validator-0                1/1     Running     0             30m
aptos-node-2-fullnode-eforge244-0       1/1     Running     0             30m
aptos-node-2-validator-0                1/1     Running     0             30m
aptos-node-3-fullnode-eforge244-0       1/1     Running     0             30m
aptos-node-3-validator-0                1/1     Running     0             30m
aptos-node-4-fullnode-eforge244-0       1/1     Running     0             30m
aptos-node-4-validator-0                1/1     Running     0             30m
aptos-node-5-validator-0                1/1     Running     0             30m
aptos-node-6-validator-0                1/1     Running     0             30m
forge-testnet-deployer-lddjf            0/1     Completed   0             30m
genesis-aptos-genesis-eforge244-wbpt8   0/1     Completed   0             30m

```
* [Grafana dashboard](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-e2e-pr-16313&var-metrics_source=All&var-chain_name=forge-1&from=1744869110000&to=1744870973000)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-e2e-pr-16313%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-e2e-pr-16313%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1744869110000&end=1744870973000)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16313%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16313%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22startTime%22%3A%20%222025-04-17T05%3A51%3A50.135382%2B00%3A00%22%2C%20%22endTime%22%3A%20%222025-04-17T06%3A22%3A53.243274%2B00%3A00%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1744869110&until=1744870973&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16313&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1744869110&until=1744870973&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16313&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14508828320)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-e2e -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-17T18:52:48Z
body: ### Forge is running suite `realistic_env_max_load` on `0907b12753e29f680e192fa3ef730864eecd7838`
* [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-e2e-pr-16313&var-metrics_source=All&var-chain_name=forge-1&refresh=10s&from=now-15m&to=now)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-e2e-pr-16313%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-e2e-pr-16313%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16313%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16313%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22quickRange%22%3A%20%2230m%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16313&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16313&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14522436870)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-e2e -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-17T18:52:52Z
body: ### Forge is running suite `compat` on `463dda939a7bdc1266946e824a1eb309f300eaf8` ==> `0907b12753e29f680e192fa3ef730864eecd7838`
* [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-compat-pr-16313&var-metrics_source=All&var-chain_name=forge-0&refresh=10s&from=now-15m&to=now)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-compat-pr-16313%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-compat-pr-16313%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16313%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16313%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22quickRange%22%3A%20%2230m%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16313&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16313&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14522436870)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-compat -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-17T18:52:53Z
body: ### Forge is running suite `framework_upgrade` on `463dda939a7bdc1266946e824a1eb309f300eaf8` ==> `0907b12753e29f680e192fa3ef730864eecd7838`
* [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-framework-upgrade-pr-16313&var-metrics_source=All&var-chain_name=forge-0&refresh=10s&from=now-15m&to=now)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-framework-upgrade-pr-16313%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-framework-upgrade-pr-16313%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16313%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16313%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22quickRange%22%3A%20%2230m%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16313&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16313&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14522436870)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-framework-upgrade -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-17T19:05:28Z
body: ### :x: Forge suite `realistic_env_max_load` failure on `0907b12753e29f680e192fa3ef730864eecd7838`
```
two traffics test: inner traffic : committed: 10016.85 txn/s, submitted: 10019.87 txn/s, expired: 3.02 txn/s, latency: 3949.53 ms, (p50: 3000 ms, p70: 3300, p90: 3900 ms, p99: 32100 ms), latency samples: 3808611
two traffics test : committed: 99.97 txn/s, latency: 1618.72 ms, (p50: 1400 ms, p70: 1600, p90: 2000 ms, p99: 5000 ms), latency samples: 1720
Latency breakdown for phase 0: ["MempoolToBlockCreation: max: 3.025, avg: 2.626", "ConsensusProposalToOrdered: max: 0.346, avg: 0.336", "ConsensusOrderedToCommit: max: 0.441, avg: 0.412", "ConsensusProposalToCommit: max: 0.780, avg: 0.747"]
Test Failed: check for success

Caused by:
    "MempoolToBlockCreation" metric violated threshold of 2.85, max_breach_pct: 5, breach_pct: 23 

Stack backtrace:
   0: anyhow::error::<impl anyhow::Error>::msg
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/anyhow-1.0.89/src/error.rs:85:36
   1: aptos_forge::success_criteria::MetricsThreshold::ensure_metrics_threshold
   2: aptos_forge::success_criteria::LatencyBreakdownThreshold::ensure_threshold
             at ./testsuite/forge/src/success_criteria.rs:157:13
   3: aptos_forge::success_criteria::SuccessCriteriaChecker::check_for_success::{{closure}}
             at ./testsuite/forge/src/success_criteria.rs:315:13
   4: aptos_forge::interface::network::NetworkContext::check_for_success::{{closure}}
             at ./testsuite/forge/src/interface/network.rs:112:10
   5: <dyn aptos_testcases::NetworkLoadTest as aptos_forge::interface::network::NetworkTest>::run::{{closure}}
             at ./testsuite/testcases/src/lib.rs:333:14
   6: <core::pin::Pin<P> as core::future::future::Future>::poll
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/future/future.rs:123:9
   7: <aptos_testcases::two_traffics_test::TwoTrafficsTest as aptos_forge::interface::network::NetworkTest>::run::{{closure}}
             at ./testsuite/testcases/src/two_traffics_test.rs:77:47
   8: <core::pin::Pin<P> as core::future::future::Future>::poll
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/future/future.rs:123:9
   9: <aptos_testcases::CompositeNetworkTest as aptos_forge::interface::network::NetworkTest>::run::{{closure}}
             at ./testsuite/testcases/src/lib.rs:639:37
  10: <core::pin::Pin<P> as core::future::future::Future>::poll
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/future/future.rs:123:9
  11: tokio::runtime::park::CachedParkThread::block_on::{{closure}}
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/park.rs:281:63
  12: tokio::runtime::coop::with_budget
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/coop.rs:107:5
  13: tokio::runtime::coop::budget
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/coop.rs:73:5
  14: tokio::runtime::park::CachedParkThread::block_on
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/park.rs:281:31
  15: tokio::runtime::context::blocking::BlockingRegionGuard::block_on
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/context/blocking.rs:66:9
  16: tokio::runtime::handle::Handle::block_on_inner::{{closure}}
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/handle.rs:324:22
  17: tokio::runtime::context::runtime::enter_runtime
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/context/runtime.rs:65:16
  18: tokio::runtime::handle::Handle::block_on_inner
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/handle.rs:323:9
  19: tokio::runtime::handle::Handle::block_on
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/handle.rs:302:18
  20: aptos_forge::runner::Forge<F>::run
             at ./testsuite/forge/src/runner.rs:332:50
  21: forge::run_forge_with_changelog
             at ./testsuite/forge-cli/src/main.rs:430:24
  22: forge::main
             at ./testsuite/forge-cli/src/main.rs:333:21
  23: core::ops::function::FnOnce::call_once
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/ops/function.rs:250:5
  24: std::sys_common::backtrace::__rust_begin_short_backtrace
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/sys_common/backtrace.rs:155:18
  25: std::rt::lang_start::{{closure}}
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/rt.rs:166:18
  26: core::ops::function::impls::<impl core::ops::function::FnOnce<A> for &F>::call_once
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/ops/function.rs:284:13
  27: std::panicking::try::do_call
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:552:40
  28: std::panicking::try
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:516:19
  29: std::panic::catch_unwind
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panic.rs:146:14
  30: std::rt::lang_start_internal::{{closure}}
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/rt.rs:148:48
  31: std::panicking::try::do_call
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:552:40
  32: std::panicking::try
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:516:19
  33: std::panic::catch_unwind
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panic.rs:146:14
  34: std::rt::lang_start_internal
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/rt.rs:148:20
  35: main
  36: __libc_start_main
  37: _start
Trailing Log Lines:
  32: std::panicking::try
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:516:19
  33: std::panic::catch_unwind
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panic.rs:146:14
  34: std::rt::lang_start_internal
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/rt.rs:148:20
  35: main
  36: __libc_start_main
  37: _start

=== BEGIN JUNIT ===
<?xml version="1.0" encoding="UTF-8"?>
<testsuites name="forge" tests="1" failures="1" errors="0" uuid="3998d147-3d23-433a-978a-7aec4598802c">
    <testsuite name="local" tests="1" disabled="0" errors="0" failures="1">
        <testcase name="CompositeNetworkTest(network:multi-region-network-emulation(two traffics test)) with ">
            <failure message="check for success

Caused by:
    &quot;MempoolToBlockCreation&quot; metric violated threshold of 2.85, max_breach_pct: 5, breach_pct: 23 

Stack backtrace:
   0: anyhow::error::&lt;impl anyhow::Error&gt;::msg
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/anyhow-1.0.89/src/error.rs:85:36
   1: aptos_forge::success_criteria::MetricsThreshold::ensure_metrics_threshold
   2: aptos_forge::success_criteria::LatencyBreakdownThreshold::ensure_threshold
             at ./testsuite/forge/src/success_criteria.rs:157:13
   3: aptos_forge::success_criteria::SuccessCriteriaChecker::check_for_success::{{closure}}
             at ./testsuite/forge/src/success_criteria.rs:315:13
   4: aptos_forge::interface::network::NetworkContext::check_for_success::{{closure}}
             at ./testsuite/forge/src/interface/network.rs:112:10
   5: &lt;dyn aptos_testcases::NetworkLoadTest as aptos_forge::interface::network::NetworkTest&gt;::run::{{closure}}
             at ./testsuite/testcases/src/lib.rs:333:14
   6: &lt;core::pin::Pin&lt;P&gt; as core::future::future::Future&gt;::poll
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/future/future.rs:123:9
   7: &lt;aptos_testcases::two_traffics_test::TwoTrafficsTest as aptos_forge::interface::network::NetworkTest&gt;::run::{{closure}}
             at ./testsuite/testcases/src/two_traffics_test.rs:77:47
   8: &lt;core::pin::Pin&lt;P&gt; as core::future::future::Future&gt;::poll
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/future/future.rs:123:9
   9: &lt;aptos_testcases::CompositeNetworkTest as aptos_forge::interface::network::NetworkTest&gt;::run::{{closure}}
             at ./testsuite/testcases/src/lib.rs:639:37
  10: &lt;core::pin::Pin&lt;P&gt; as core::future::future::Future&gt;::poll
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/future/future.rs:123:9
  11: tokio::runtime::park::CachedParkThread::block_on::{{closure}}
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/park.rs:281:63
  12: tokio::runtime::coop::with_budget
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/coop.rs:107:5
  13: tokio::runtime::coop::budget
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/coop.rs:73:5
  14: tokio::runtime::park::CachedParkThread::block_on
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/park.rs:281:31
  15: tokio::runtime::context::blocking::BlockingRegionGuard::block_on
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/context/blocking.rs:66:9
  16: tokio::runtime::handle::Handle::block_on_inner::{{closure}}
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/handle.rs:324:22
  17: tokio::runtime::context::runtime::enter_runtime
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/context/runtime.rs:65:16
  18: tokio::runtime::handle::Handle::block_on_inner
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/handle.rs:323:9
  19: tokio::runtime::handle::Handle::block_on
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/handle.rs:302:18
  20: aptos_forge::runner::Forge&lt;F&gt;::run
             at ./testsuite/forge/src/runner.rs:332:50
  21: forge::run_forge_with_changelog
             at ./testsuite/forge-cli/src/main.rs:430:24
  22: forge::main
             at ./testsuite/forge-cli/src/main.rs:333:21
  23: core::ops::function::FnOnce::call_once
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/ops/function.rs:250:5
  24: std::sys_common::backtrace::__rust_begin_short_backtrace
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/sys_common/backtrace.rs:155:18
  25: std::rt::lang_start::{{closure}}
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/rt.rs:166:18
  26: core::ops::function::impls::&lt;impl core::ops::function::FnOnce&lt;A&gt; for &amp;F&gt;::call_once
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/ops/function.rs:284:13
  27: std::panicking::try::do_call
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:552:40
  28: std::panicking::try
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:516:19
  29: std::panic::catch_unwind
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panic.rs:146:14
  30: std::rt::lang_start_internal::{{closure}}
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/rt.rs:148:48
  31: std::panicking::try::do_call
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:552:40
  32: std::panicking::try
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:516:19
  33: std::panic::catch_unwind
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panic.rs:146:14
  34: std::rt::lang_start_internal
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/rt.rs:148:20
  35: main
  36: __libc_start_main
  37: _start"/>
        </testcase>
    </testsuite>
</testsuites>
=== END JUNIT ===

Swarm logs can be found here: See fgi output for more information.
[2025-04-17T19:05:23Z INFO  aptos_forge::backend::k8s::cluster_helper] Deleting namespace forge-e2e-pr-16313: Some(NamespaceStatus { conditions: None, phase: Some("Terminating") })
[2025-04-17T19:05:23Z INFO  aptos_forge::backend::k8s::cluster_helper] aptos-node resources for Forge removed in namespace: forge-e2e-pr-16313

failures:
    CompositeNetworkTest
Failed to run tests:
Tests Failed

test result: FAILED. 0 passed; 1 failed; 0 filtered out

Error: Tests Failed

Stack backtrace:
   0: anyhow::error::<impl anyhow::Error>::msg
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/anyhow-1.0.89/src/error.rs:85:36
   1: aptos_forge::runner::Forge<F>::run
             at ./testsuite/forge/src/runner.rs:358:13
   2: forge::run_forge_with_changelog
             at ./testsuite/forge-cli/src/main.rs:430:24
   3: forge::main
             at ./testsuite/forge-cli/src/main.rs:333:21
   4: core::ops::function::FnOnce::call_once
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/ops/function.rs:250:5
   5: std::sys_common::backtrace::__rust_begin_short_backtrace
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/sys_common/backtrace.rs:155:18
   6: std::rt::lang_start::{{closure}}
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/rt.rs:166:18
   7: core::ops::function::impls::<impl core::ops::function::FnOnce<A> for &F>::call_once
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/ops/function.rs:284:13
   8: std::panicking::try::do_call
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:552:40
   9: std::panicking::try
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:516:19
  10: std::panic::catch_unwind
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panic.rs:146:14
  11: std::rt::lang_start_internal::{{closure}}
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/rt.rs:148:48
  12: std::panicking::try::do_call
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:552:40
  13: std::panicking::try
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:516:19
  14: std::panic::catch_unwind
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panic.rs:146:14
  15: std::rt::lang_start_internal
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/rt.rs:148:20
  16: main
  17: __libc_start_main
  18: _start
Debugging output:
NAME                                   READY   STATUS      RESTARTS   AGE
aptos-node-0-fullnode-eforge52-0       1/1     Running     0          11m
aptos-node-0-validator-0               1/1     Running     0          11m
aptos-node-1-fullnode-eforge52-0       1/1     Running     0          11m
aptos-node-1-validator-0               1/1     Running     0          11m
aptos-node-2-fullnode-eforge52-0       1/1     Running     0          11m
aptos-node-2-validator-0               1/1     Running     0          11m
aptos-node-3-fullnode-eforge52-0       1/1     Running     0          11m
aptos-node-3-validator-0               1/1     Running     0          11m
aptos-node-4-fullnode-eforge52-0       1/1     Running     0          11m
aptos-node-4-validator-0               1/1     Running     0          11m
aptos-node-5-validator-0               1/1     Running     0          11m
aptos-node-6-validator-0               1/1     Running     0          11m
genesis-aptos-genesis-eforge52-hbckp   0/1     Completed   0          11m

```
* [Grafana dashboard](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-e2e-pr-16313&var-metrics_source=All&var-chain_name=forge-1&from=1744915975000&to=1744916726000)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-e2e-pr-16313%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-e2e-pr-16313%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1744915975000&end=1744916726000)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16313%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16313%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22startTime%22%3A%20%222025-04-17T18%3A52%3A55.113328%2B00%3A00%22%2C%20%22endTime%22%3A%20%222025-04-17T19%3A05%3A26.334067%2B00%3A00%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1744915975&until=1744916726&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16313&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1744915975&until=1744916726&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16313&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14522436870)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-e2e -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-17T19:06:03Z
body: ### :white_check_mark: Forge suite `framework_upgrade` success on `463dda939a7bdc1266946e824a1eb309f300eaf8` ==> `0907b12753e29f680e192fa3ef730864eecd7838`
```
Compatibility test results for 463dda939a7bdc1266946e824a1eb309f300eaf8 ==> 0907b12753e29f680e192fa3ef730864eecd7838 (PR)
Upgrade the nodes to version: 0907b12753e29f680e192fa3ef730864eecd7838
framework_upgrade::framework-upgrade::full-framework-upgrade : committed: 1744.77 txn/s, submitted: 1749.90 txn/s, failed submission: 5.13 txn/s, expired: 5.13 txn/s, latency: 1680.07 ms, (p50: 1500 ms, p70: 1800, p90: 2400 ms, p99: 3400 ms), latency samples: 156541
framework_upgrade::framework-upgrade::full-framework-upgrade : committed: 1696.60 txn/s, submitted: 1702.98 txn/s, failed submission: 6.38 txn/s, expired: 6.38 txn/s, latency: 1713.64 ms, (p50: 1500 ms, p70: 1800, p90: 2600 ms, p99: 3600 ms), latency samples: 154300
5. check swarm health
Compatibility test for 463dda939a7bdc1266946e824a1eb309f300eaf8 ==> 0907b12753e29f680e192fa3ef730864eecd7838 passed
Upgrade the remaining nodes to version: 0907b12753e29f680e192fa3ef730864eecd7838
framework_upgrade::framework-upgrade::full-framework-upgrade : committed: 1643.04 txn/s, submitted: 1647.91 txn/s, failed submission: 4.69 txn/s, expired: 4.87 txn/s, latency: 1804.22 ms, (p50: 1600 ms, p70: 1900, p90: 2700 ms, p99: 3700 ms), latency samples: 147227
Test Ok
```
* [Grafana dashboard](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-framework-upgrade-pr-16313&var-metrics_source=All&var-chain_name=forge-0&from=1744915980000&to=1744916761000)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-framework-upgrade-pr-16313%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-framework-upgrade-pr-16313%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1744915980000&end=1744916761000)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16313%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16313%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22startTime%22%3A%20%222025-04-17T18%3A53%3A00.578601%2B00%3A00%22%2C%20%22endTime%22%3A%20%222025-04-17T19%3A06%3A01.336894%2B00%3A00%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1744915980&until=1744916761&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16313&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1744915980&until=1744916761&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16313&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14522436870)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-framework-upgrade -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-17T19:37:09Z
body: ### Forge is running suite `compat` on `463dda939a7bdc1266946e824a1eb309f300eaf8` ==> `cd8b0918c1bc2ab56285af45bf02f0fa69e6f5d4`
* [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-compat-pr-16313&var-metrics_source=All&var-chain_name=forge-0&refresh=10s&from=now-15m&to=now)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-compat-pr-16313%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-compat-pr-16313%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16313%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16313%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22quickRange%22%3A%20%2230m%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16313&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16313&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14523199145)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-compat -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-17T19:37:09Z
body: ### Forge is running suite `realistic_env_max_load` on `cd8b0918c1bc2ab56285af45bf02f0fa69e6f5d4`
* [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-e2e-pr-16313&var-metrics_source=All&var-chain_name=forge-1&refresh=10s&from=now-15m&to=now)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-e2e-pr-16313%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-e2e-pr-16313%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16313%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16313%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22quickRange%22%3A%20%2230m%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16313&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16313&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14523199145)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-e2e -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-17T19:37:16Z
body: ### Forge is running suite `framework_upgrade` on `463dda939a7bdc1266946e824a1eb309f300eaf8` ==> `cd8b0918c1bc2ab56285af45bf02f0fa69e6f5d4`
* [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-framework-upgrade-pr-16313&var-metrics_source=All&var-chain_name=forge-0&refresh=10s&from=now-15m&to=now)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-framework-upgrade-pr-16313%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-framework-upgrade-pr-16313%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16313%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16313%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22quickRange%22%3A%20%2230m%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16313&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16313&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14523199145)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-framework-upgrade -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-17T19:51:04Z
body: ### :white_check_mark: Forge suite `realistic_env_max_load` success on `cd8b0918c1bc2ab56285af45bf02f0fa69e6f5d4`
```
two traffics test: inner traffic : committed: 12166.38 txn/s, submitted: 12170.53 txn/s, expired: 4.16 txn/s, latency: 3227.81 ms, (p50: 3000 ms, p70: 3300, p90: 3600 ms, p99: 5400 ms), latency samples: 4625940
two traffics test : committed: 99.99 txn/s, latency: 2121.30 ms, (p50: 1600 ms, p70: 2000, p90: 2400 ms, p99: 22200 ms), latency samples: 1760
Latency breakdown for phase 0: ["MempoolToBlockCreation: max: 1.938, avg: 1.666", "ConsensusProposalToOrdered: max: 0.316, avg: 0.310", "ConsensusOrderedToCommit: max: 0.495, avg: 0.420", "ConsensusProposalToCommit: max: 0.802, avg: 0.730"]
Max non-epoch-change gap was: 1 rounds at version 44021 (avg 0.00) [limit 4], 1.90s no progress at version 44021 (avg 0.21s) [limit 15].
Max epoch-change gap was: 0 rounds at version 0 (avg 0.00) [limit 4], 0.94s no progress at version 4349872 (avg 0.86s) [limit 16].
Test Ok
```
* [Grafana dashboard](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-e2e-pr-16313&var-metrics_source=All&var-chain_name=forge-1&from=1744918636000&to=1744919461000)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-e2e-pr-16313%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-e2e-pr-16313%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1744918636000&end=1744919461000)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16313%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16313%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22startTime%22%3A%20%222025-04-17T19%3A37%3A16.933899%2B00%3A00%22%2C%20%22endTime%22%3A%20%222025-04-17T19%3A51%3A01.951680%2B00%3A00%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1744918636&until=1744919461&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16313&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1744918636&until=1744919461&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16313&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14523199145)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-e2e -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-17T19:51:10Z
body: ### :white_check_mark: Forge suite `framework_upgrade` success on `463dda939a7bdc1266946e824a1eb309f300eaf8` ==> `cd8b0918c1bc2ab56285af45bf02f0fa69e6f5d4`
```
Forge report malformed: Expecting ',' delimiter: line 18 column 6 (char 480)
'{\n  "metrics": [\n    {\n      "test_name": "framework_upgrade::framework-upgrade::full-framework-upgrade",\n      "metric": "submitted_txn",\n      "value": 139702.0\n    },\n    {\n      "test_name": "framework_upgrade::framework-upgrade::full-framework-upgrade",\n      "metric": "expired_txn",\n      "value": 500.0\n    },\n    {\n      "test_name": "framework_upgrade::framework-upgrade::full-framework-upgrade",\n      "metric": "avg_tps",\n      "value": 1573.2070955086501\n    },\n[2025-04-17T19:51:05Z INFO  aptos_forge::report] Test Ok\n    {\n      "test_name": "framework_upgrade::framework-upgrade::full-framework-upgrade",\n      "metric": "avg_latency",\n      "value": 2487.119272711599\n    },\n    {\n      "test_name": "framework_upgrade::framework-upgrade::full-framework-upgrade",\n      "metric": "p50_latency",\n      "value": 1800.0\n    },\n    {\n      "test_name": "framework_upgrade::framework-upgrade::full-framework-upgrade",\n      "metric": "p90_latency",\n      "value": 3300.0\n    },\n    {\n      "test_name": "framework_upgrade::framework-upgrade::full-framework-upgrade",\n      "metric": "p99_latency",\n      "value": 13100.0\n    },\n    {\n      "test_name": "framework_upgrade::framework-upgrade::full-framework-upgrade",\n      "metric": "submitted_txn",\n      "value": 155660.0\n    },\n    {\n      "test_name": "framework_upgrade::framework-upgrade::full-framework-upgrade",\n      "metric": "expired_txn",\n      "value": 753.0\n    },\n    {\n      "test_name": "framework_upgrade::framework-upgrade::full-framework-upgrade",\n      "metric": "avg_tps",\n      "value": 1704.172712257373\n    },\n    {\n      "test_name": "framework_upgrade::framework-upgrade::full-framework-upgrade",\n      "metric": "avg_latency",\n      "value": 1708.919028836657\n    },\n    {\n      "test_name": "framework_upgrade::framework-upgrade::full-framework-upgrade",\n      "metric": "p50_latency",\n      "value": 1600.0\n    },\n    {\n      "test_name": "framework_upgrade::framework-upgrade::full-framework-upgrade",\n      "metric": "p90_latency",\n      "value": 2400.0\n    },\n    {\n      "test_name": "framework_upgrade::framework-upgrade::full-framework-upgrade",\n      "metric": "p99_latency",\n      "value": 3300.0\n    },\n    {\n      "test_name": "framework_upgrade::framework-upgrade::full-framework-upgrade",\n      "metric": "submitted_txn",\n      "value": 109782.0\n    },\n    {\n      "test_name": "framework_upgrade::framework-upgrade::full-framework-upgrade",\n      "metric": "expired_txn",\n      "value": 340.0\n    },\n    {\n      "test_name": "framework_upgrade::framework-upgrade::full-framework-upgrade",\n      "metric": "avg_tps",\n      "value": 1213.650195735411\n    },\n    {\n      "test_name": "framework_upgrade::framework-upgrade::full-framework-upgrade",\n      "metric": "avg_latency",\n      "value": 2465.7776173681036\n    },\n    {\n      "test_name": "framework_upgrade::framework-upgrade::full-framework-upgrade",\n      "metric": "p50_latency",\n      "value": 1500.0\n    },\n    {\n      "test_name": "framework_upgrade::framework-upgrade::full-framework-upgrade",\n      "metric": "p90_latency",\n      "value": 3000.0\n    },\n    {\n      "test_name": "framework_upgrade::framework-upgrade::full-framework-upgrade",\n      "metric": "p99_latency",\n      "value": 12600.0\n    }\n  ],\n  "text": "Compatibility test results for 463dda939a7bdc1266946e824a1eb309f300eaf8 ==> cd8b0918c1bc2ab56285af45bf02f0fa69e6f5d4 (PR)\\nUpgrade the nodes to version: cd8b0918c1bc2ab56285af45bf02f0fa69e6f5d4\\nframework_upgrade::framework-upgrade::full-framework-upgrade : committed: 1573.21 txn/s, submitted: 1578.86 txn/s, failed submission: 5.65 txn/s, expired: 5.65 txn/s, latency: 2487.12 ms, (p50: 1800 ms, p70: 2100, p90: 3300 ms, p99: 13100 ms), latency samples: 139202\\nframework_upgrade::framework-upgrade::full-framework-upgrade : committed: 1704.17 txn/s, submitted: 1712.46 txn/s, failed submission: 8.14 txn/s, expired: 8.28 txn/s, latency: 1708.92 ms, (p50: 1600 ms, p70: 2000, p90: 2400 ms, p99: 3300 ms), latency samples: 154907\\n5. check swarm health\\nCompatibility test for 463dda939a7bdc1266946e824a1eb309f300eaf8 ==> cd8b0918c1bc2ab56285af45bf02f0fa69e6f5d4 passed\\nUpgrade the remaining nodes to version: cd8b0918c1bc2ab56285af45bf02f0fa69e6f5d4\\nframework_upgrade::framework-upgrade::full-framework-upgrade : committed: 1213.65 txn/s, submitted: 1217.42 txn/s, failed submission: 3.77 txn/s, expired: 3.77 txn/s, latency: 2465.78 ms, (p50: 1500 ms, p70: 1900, p90: 3000 ms, p99: 12600 ms), latency samples: 109442\\nTest Ok"\n}'
Trailing Log Lines:
Compatibility test results for 463dda939a7bdc1266946e824a1eb309f300eaf8 ==> cd8b0918c1bc2ab56285af45bf02f0fa69e6f5d4 (PR)
Upgrade the nodes to version: cd8b0918c1bc2ab56285af45bf02f0fa69e6f5d4
framework_upgrade::framework-upgrade::full-framework-upgrade : committed: 1573.21 txn/s, submitted: 1578.86 txn/s, failed submission: 5.65 txn/s, expired: 5.65 txn/s, latency: 2487.12 ms, (p50: 1800 ms, p70: 2100, p90: 3300 ms, p99: 13100 ms), latency samples: 139202
framework_upgrade::framework-upgrade::full-framework-upgrade : committed: 1704.17 txn/s, submitted: 1712.46 txn/s, failed submission: 8.14 txn/s, expired: 8.28 txn/s, latency: 1708.92 ms, (p50: 1600 ms, p70: 2000, p90: 2400 ms, p99: 3300 ms), latency samples: 154907
5. check swarm health
Compatibility test for 463dda939a7bdc1266946e824a1eb309f300eaf8 ==> cd8b0918c1bc2ab56285af45bf02f0fa69e6f5d4 passed
Upgrade the remaining nodes to version: cd8b0918c1bc2ab56285af45bf02f0fa69e6f5d4
framework_upgrade::framework-upgrade::full-framework-upgrade : committed: 1213.65 txn/s, submitted: 1217.42 txn/s, failed submission: 3.77 txn/s, expired: 3.77 txn/s, latency: 2465.78 ms, (p50: 1500 ms, p70: 1900, p90: 3000 ms, p99: 12600 ms), latency samples: 109442
Test Ok

=== BEGIN JUNIT ===
<?xml version="1.0" encoding="UTF-8"?>
<testsuites name="forge" tests="1" failures="0" errors="0" uuid="707f8f11-8ee4-4d2c-9ea9-711e53c4d30f">
    <testsuite name="local" tests="1" disabled="0" errors="0" failures="0">
        <testcase name="framework_upgrade::framework-upgrade">
        </testcase>
    </testsuite>
</testsuites>
=== END JUNIT ===
[2025-04-17T19:51:05Z INFO  aptos_forge::backend::k8s::cluster_helper] Deleting namespace forge-framework-upgrade-pr-16313: Some(NamespaceStatus { conditions: None, phase: Some("Terminating") })
[2025-04-17T19:51:05Z INFO  aptos_forge::backend::k8s::cluster_helper] aptos-node resources for Forge removed in namespace: forge-framework-upgrade-pr-16313

test result: ok. 1 passed; 0 failed; 0 filtered out

Debugging output:
NAME                                   READY   STATUS      RESTARTS   AGE
aptos-node-0-validator-0               1/1     Running     0          10m
aptos-node-1-validator-0               1/1     Running     0          10m
aptos-node-2-validator-0               1/1     Running     0          3m14s
aptos-node-3-validator-0               1/1     Running     0          2m40s
forge-testnet-deployer-xmj6h           0/1     Completed   0          13m
genesis-aptos-genesis-eforge84-5ztqz   0/1     Completed   0          12m

```
* [Grafana dashboard](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-framework-upgrade-pr-16313&var-metrics_source=All&var-chain_name=forge-0&from=1744918644000&to=1744919468000)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-framework-upgrade-pr-16313%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-framework-upgrade-pr-16313%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1744918644000&end=1744919468000)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16313%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16313%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22startTime%22%3A%20%222025-04-17T19%3A37%3A24.063492%2B00%3A00%22%2C%20%22endTime%22%3A%20%222025-04-17T19%3A51%3A08.470986%2B00%3A00%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1744918644&until=1744919468&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16313&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1744918644&until=1744919468&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16313&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14523199145)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-framework-upgrade -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-17T20:08:22Z
body: ### :white_check_mark: Forge suite `compat` success on `463dda939a7bdc1266946e824a1eb309f300eaf8` ==> `cd8b0918c1bc2ab56285af45bf02f0fa69e6f5d4`
```
Compatibility test results for 463dda939a7bdc1266946e824a1eb309f300eaf8 ==> cd8b0918c1bc2ab56285af45bf02f0fa69e6f5d4 (PR)
1. Check liveness of validators at old version: 463dda939a7bdc1266946e824a1eb309f300eaf8
compatibility::simple-validator-upgrade::liveness-check : committed: 3655.09 txn/s, submitted: 3655.26 txn/s, expired: 0.17 txn/s, latency: 3573.09 ms, (p50: 3800 ms, p70: 4100, p90: 4400 ms, p99: 5000 ms), latency samples: 304786
2. Upgrading first Validator to new version: cd8b0918c1bc2ab56285af45bf02f0fa69e6f5d4
compatibility::simple-validator-upgrade::single-validator-upgrade : committed: 2765.10 txn/s, latency: 11963.01 ms, (p50: 13600 ms, p70: 13800, p90: 13900 ms, p99: 14000 ms), latency samples: 103940
3. Upgrading rest of first batch to new version: cd8b0918c1bc2ab56285af45bf02f0fa69e6f5d4
compatibility::simple-validator-upgrade::half-validator-upgrade : committed: 2870.62 txn/s, latency: 11592.34 ms, (p50: 13100 ms, p70: 13500, p90: 13700 ms, p99: 13800 ms), latency samples: 108360
4. upgrading second batch to new version: cd8b0918c1bc2ab56285af45bf02f0fa69e6f5d4
compatibility::simple-validator-upgrade::rest-validator-upgrade : committed: 5048.99 txn/s, latency: 6817.91 ms, (p50: 7300 ms, p70: 7700, p90: 7900 ms, p99: 8100 ms), latency samples: 177440
5. check swarm health
Compatibility test for 463dda939a7bdc1266946e824a1eb309f300eaf8 ==> cd8b0918c1bc2ab56285af45bf02f0fa69e6f5d4 passed
Test Ok
```
* [Grafana dashboard](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-compat-pr-16313&var-metrics_source=All&var-chain_name=forge-0&from=1744918638000&to=1744920500000)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-compat-pr-16313%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-compat-pr-16313%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1744918638000&end=1744920500000)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16313%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16313%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22startTime%22%3A%20%222025-04-17T19%3A37%3A18.348810%2B00%3A00%22%2C%20%22endTime%22%3A%20%222025-04-17T20%3A08%3A20.791029%2B00%3A00%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1744918638&until=1744920500&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16313&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1744918638&until=1744920500&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16313&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14523199145)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-compat -->
author_association: CONTRIBUTOR