Repository: aptos-labs/aptos-core
Pull request #16555: Fix transactional tests when aptos-experimental uses some new FV features
Author: login: igor-aptos
Created: 2025-05-07T21:43:35Z | Merged: 2025-05-08T22:26:45Z

…

## Description
<!-- Please include a summary of the change, including which issue it fixes or what feature it adds. Include relevant motivation, context and documentation as appropriate. List dependencies that are required for this change, if any. -->

## How Has This Been Tested?
<!--
- Please ensure that the functionality introduced by this change is well tested and verified to work as expected.
- Ensure tests cover both happy and unhappy paths.
- List and link relevant tests.
-->

## Key Areas to Review
<!--
- Identify any critical parts of the code that require special attention or understanding. Explain why these parts are crucial to the functionality or architecture of the project.
- Point out any areas where complex logic has been implemented. Provide a brief explanation of the logic and your approach to make it easier for reviewers to follow.
- Highlight any areas where you are particularly concerned or unsure about the code's impact on the change. This can include potential performance or security issues, or compatibility with existing features.
-->

## Type of Change
- [ ] New feature
- [ ] Bug fix
- [ ] Breaking change
- [ ] Performance improvement
- [ ] Refactoring
- [ ] Dependency update
- [ ] Documentation update
- [ ] Tests

## Which Components or Systems Does This Change Impact?
- [ ] Validator Node
- [ ] Full Node (API, Indexer, etc.)
- [ ] Move/Aptos Virtual Machine
- [ ] Aptos Framework
- [ ] Aptos CLI/SDK
- [ ] Developer Infrastructure
- [ ] Move Compiler
- [ ] Other (specify)

## Checklist
- [ ] I have read and followed the [CONTRIBUTING](https://github.com/aptos-labs/aptos-core/blob/main/CONTRIBUTING.md) doc
- [ ] I have performed a self-review of my own code
- [ ] I have commented my code, particularly in hard-to-understand areas
- [ ] I identified and added all stakeholders and component owners affected by this change as reviewers
- [ ] I tested both happy and unhappy path of the functionality
- [ ] I have made corresponding changes to the documentation

<!-- Thank you for your contribution! -->



Diff:
diff --git a/aptos-move/aptos-transactional-test-harness/src/aptos_test_harness.rs b/aptos-move/aptos-transactional-test-harness/src/aptos_test_harness.rs
index d2e6801a36d..61c512961d2 100644
--- a/aptos-move/aptos-transactional-test-harness/src/aptos_test_harness.rs
+++ b/aptos-move/aptos-transactional-test-harness/src/aptos_test_harness.rs
@@ -292,66 +292,76 @@ fn panic_missing_private_key(cmd_name: &str) -> ! {
     )
 }
 
-static APTOS_FRAMEWORK_FILES: Lazy<Vec<String>> = Lazy::new(|| {
-    aptos_cached_packages::head_release_bundle()
-        .files()
-        .unwrap()
-});
-
-static PRECOMPILED_APTOS_FRAMEWORK_V2: Lazy<PrecompiledFilesModules> = Lazy::new(|| {
+fn compile_framework_sources(
+    all_source_files: Vec<String>,
+    language_version: Option<LanguageVersion>,
+) -> PrecompiledFilesModules {
     let named_address_mapping_strings: Vec<String> = aptos_framework::named_addresses()
         .iter()
         .map(|(string, num_addr)| format!("{}={}", string, num_addr))
         .collect();
 
-    let is_experimental =
-        |f: &String| f.contains("aptos-move/framework/aptos-experimental/sources");
-
-    // aptos-experimental can be using latest language features, while others cannot.
-    // So we need to compile it twice, once without aptos-experimental, and once with.
-    // (in the second pass, we need to provide both, in case aptos-experimental depends on other modules)
-
-    let all_sources = aptos_cached_packages::head_release_bundle()
-        .files()
-        .unwrap();
-    let prod_sources = all_sources
-        .iter()
-        .filter(|f| !is_experimental(f))
-        .cloned()
-        .collect::<Vec<_>>();
-
-    let mut options = move_compiler_v2::Options {
-        sources: prod_sources,
+    let options = move_compiler_v2::Options {
+        sources: all_source_files.clone(),
         dependencies: vec![],
         named_address_mapping: named_address_mapping_strings.clone(),
         known_attributes: aptos_framework::extended_checks::get_all_attribute_names().clone(),
-        language_version: None,
+        language_version,
         ..move_compiler_v2::Options::default()
     };
 
-    let (_global_env, mut modules) = move_compiler_v2::run_move_compiler_to_stderr(options.clone())
+    let (_global_env, modules) = move_compiler_v2::run_move_compiler_to_stderr(options.clone())
         .expect("framework compilation succeeds");
 
-    options.sources = all_sources;
-    options.language_version = Some(LanguageVersion::latest());
+    PrecompiledFilesModules::new(all_source_files, modules)
+}
 
-    let (experimental_global_env, experimental_modules) =
-        move_compiler_v2::run_move_compiler_to_stderr(options)
-            .expect("aptos-experimental compilation succeeds");
+// aptos-experimental can be using latest language features, while others cannot.
+// So we need to compile it twice, once without aptos-experimental, and once with.
 
-    for cur_module in experimental_modules {
-        let file_name = experimental_global_env.get_file(
-            experimental_global_env
-                .get_file_id(cur_module.loc().file_hash())
-                .unwrap(),
-        );
-        if is_experimental(&file_name.to_str().unwrap().to_string()) {
-            modules.push(cur_module);
-        }
-    }
-    PrecompiledFilesModules::new(APTOS_FRAMEWORK_FILES.clone(), modules)
+static PRECOMPILED_APTOS_FRAMEWORK_V2: Lazy<PrecompiledFilesModules> = Lazy::new(|| {
+    compile_framework_sources(
+        aptos_cached_packages::head_release_bundle()
+            .files()
+            .unwrap()
+            .iter()
+            .filter(|f| !f.contains("aptos-move/framework/aptos-experimental/sources"))
+            .cloned()
+            .collect::<Vec<_>>(),
+        None,
+    )
 });
 
+static PRECOMPILED_APTOS_FRAMEWORK_V2_WITH_EXPERIMENTAL: Lazy<PrecompiledFilesModules> =
+    Lazy::new(|| {
+        let named_address_mapping_strings: Vec<String> = aptos_framework::named_addresses()
+            .iter()
+            .map(|(string, num_addr)| format!("{}={}", string, num_addr))
+            .collect();
+
+        // aptos-experimental can be using latest language features, while others cannot.
+        // So we need to compile it twice, once without aptos-experimental, and once with.
+        // (in the second pass, we need to provide both, in case aptos-experimental depends on other modules)
+
+        let all_sources = aptos_cached_packages::head_release_bundle()
+            .files()
+            .unwrap();
+
+        let options = move_compiler_v2::Options {
+            sources: all_sources.clone(),
+            dependencies: vec![],
+            named_address_mapping: named_address_mapping_strings.clone(),
+            known_attributes: aptos_framework::extended_checks::get_all_attribute_names().clone(),
+            language_version: Some(LanguageVersion::latest()),
+            ..move_compiler_v2::Options::default()
+        };
+
+        let (_global_env, modules) = move_compiler_v2::run_move_compiler_to_stderr(options.clone())
+            .expect("framework compilation succeeds");
+
+        PrecompiledFilesModules::new(all_sources, modules)
+    });
+
 /**
  * Test Adapter Implementation
  */
@@ -1060,10 +1070,14 @@ fn render_events(events: &[ContractEvent]) -> Option<String> {
     }
 }
 
-fn precompiled_v2_stdlib() -> &'static PrecompiledFilesModules {
+fn precompiled_v2_framework() -> &'static PrecompiledFilesModules {
     &PRECOMPILED_APTOS_FRAMEWORK_V2
 }
 
+fn precompiled_v2_framework_with_experimental() -> &'static PrecompiledFilesModules {
+    &PRECOMPILED_APTOS_FRAMEWORK_V2_WITH_EXPERIMENTAL
+}
+
 pub fn run_aptos_test(path: &Path) -> Result<(), Box<dyn std::error::Error>> {
     run_aptos_test_with_config(path, TestRunConfig::CompilerV2 {
         language_version: LanguageVersion::default(),
@@ -1076,7 +1090,18 @@ pub fn run_aptos_test_with_config(
     config: TestRunConfig,
 ) -> Result<(), Box<dyn std::error::Error>> {
     let suffix = Some(EXP_EXT.to_owned());
-    let v2_lib = precompiled_v2_stdlib();
+    let v2_lib = match &config {
+        // only latest language version can use experimental framework
+        TestRunConfig::CompilerV2 {
+            language_version, ..
+        } => {
+            if language_version == &LanguageVersion::latest() {
+                precompiled_v2_framework_with_experimental()
+            } else {
+                precompiled_v2_framework()
+            }
+        },
+    };
     set_paranoid_type_checks(true);
     run_test_impl::<AptosTestAdapter>(config, path, v2_lib, &suffix)
 }
diff --git a/aptos-move/framework/aptos-experimental/doc/test_function_values.md b/aptos-move/framework/aptos-experimental/doc/test_function_values.md
index ebc2cb05824..9e047f64825 100644
--- a/aptos-move/framework/aptos-experimental/doc/test_function_values.md
+++ b/aptos-move/framework/aptos-experimental/doc/test_function_values.md
@@ -5,6 +5,7 @@
 
 
 
+-  [Struct `Funcs`](#0x7_test_function_values_Funcs)
 -  [Function `transfer_and_create_account`](#0x7_test_function_values_transfer_and_create_account)
 
 
@@ -12,6 +13,33 @@
 
 
 
+<a id="0x7_test_function_values_Funcs"></a>
+
+## Struct `Funcs`
+
+
+
+<pre><code><b>struct</b> <a href="test_function_values.md#0x7_test_function_values_Funcs">Funcs</a>
+</code></pre>
+
+
+
+<details>
+<summary>Fields</summary>
+
+
+<dl>
+<dt>
+<code>f: |u64|u64 <b>has</b> <b>copy</b> + drop</code>
+</dt>
+<dd>
+
+</dd>
+</dl>
+
+
+</details>
+
 <a id="0x7_test_function_values_transfer_and_create_account"></a>
 
 ## Function `transfer_and_create_account`
diff --git a/aptos-move/framework/aptos-experimental/sources/test_function_values.move b/aptos-move/framework/aptos-experimental/sources/test_function_values.move
index 3ceacc721b8..a84d50b4416 100644
--- a/aptos-move/framework/aptos-experimental/sources/test_function_values.move
+++ b/aptos-move/framework/aptos-experimental/sources/test_function_values.move
@@ -1,4 +1,7 @@
 module aptos_experimental::test_function_values {
+    struct Funcs {
+        f: |u64| u64 has drop + copy,
+    }
 
     fun transfer_and_create_account(some_f: |u64|u64): u64 {
         some_f(3)


Reviews:
user: login: georgemitenkov
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-05-08T11:36:49Z
user: login: igor-aptos
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-08T15:48:03Z
user: login: igor-aptos
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-08T15:48:30Z
user: login: igor-aptos
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-08T19:00:39Z
user: login: wrwg
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-05-08T19:05:35Z

Review comments:
diff_hunk: @@ -1,4 +1,7 @@
 module aptos_experimental::test_function_values {
+    struct Funcs {
path: aptos-move/framework/aptos-experimental/sources/test_function_values.move
user: login: georgemitenkov
body: For my understanding of the context: this is a test module to check that it is publishable, right? So we specifically do not want `#[test_only]`
created_at: 2025-05-08T11:32:07Z
author_association: CONTRIBUTOR
diff_hunk: @@ -320,37 +314,50 @@ static PRECOMPILED_APTOS_FRAMEWORK_V2: Lazy<PrecompiledFilesModules> = Lazy::new
         .cloned()
         .collect::<Vec<_>>();
 
-    let mut options = move_compiler_v2::Options {
-        sources: prod_sources,
+    let options = move_compiler_v2::Options {
+        sources: prod_sources.clone(),
         dependencies: vec![],
         named_address_mapping: named_address_mapping_strings.clone(),
         known_attributes: aptos_framework::extended_checks::get_all_attribute_names().clone(),
         language_version: None,
path: aptos-move/aptos-transactional-test-harness/src/aptos_test_harness.rs
user: login: georgemitenkov
body: Why this is `None` here, should it be using `LanguageVersion::default` or will it unwrap on none to default?
created_at: 2025-05-08T11:35:49Z
author_association: CONTRIBUTOR
diff_hunk: @@ -320,37 +314,50 @@ static PRECOMPILED_APTOS_FRAMEWORK_V2: Lazy<PrecompiledFilesModules> = Lazy::new
         .cloned()
         .collect::<Vec<_>>();
 
-    let mut options = move_compiler_v2::Options {
-        sources: prod_sources,
+    let options = move_compiler_v2::Options {
+        sources: prod_sources.clone(),
         dependencies: vec![],
         named_address_mapping: named_address_mapping_strings.clone(),
         known_attributes: aptos_framework::extended_checks::get_all_attribute_names().clone(),
         language_version: None,
         ..move_compiler_v2::Options::default()
     };
 
-    let (_global_env, mut modules) = move_compiler_v2::run_move_compiler_to_stderr(options.clone())
+    let (_global_env, modules) = move_compiler_v2::run_move_compiler_to_stderr(options.clone())
         .expect("framework compilation succeeds");
 
-    options.sources = all_sources;
-    options.language_version = Some(LanguageVersion::latest());
+    PrecompiledFilesModules::new(prod_sources, modules)
+});
+
+static PRECOMPILED_APTOS_FRAMEWORK_V2_WITH_EXPERIMENTAL: Lazy<PrecompiledFilesModules> =
path: aptos-move/aptos-transactional-test-harness/src/aptos_test_harness.rs
user: login: georgemitenkov
body: nit: both lazy initializers are the same moduo language version and experimental filter, can we factor these out into a function that can be called here and for framework only?
created_at: 2025-05-08T11:36:41Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1,4 +1,7 @@
 module aptos_experimental::test_function_values {
+    struct Funcs {
path: aptos-move/framework/aptos-experimental/sources/test_function_values.move
user: login: igor-aptos
body: yes.
created_at: 2025-05-08T15:48:02Z
author_association: CONTRIBUTOR
diff_hunk: @@ -320,37 +314,50 @@ static PRECOMPILED_APTOS_FRAMEWORK_V2: Lazy<PrecompiledFilesModules> = Lazy::new
         .cloned()
         .collect::<Vec<_>>();
 
-    let mut options = move_compiler_v2::Options {
-        sources: prod_sources,
+    let options = move_compiler_v2::Options {
+        sources: prod_sources.clone(),
         dependencies: vec![],
         named_address_mapping: named_address_mapping_strings.clone(),
         known_attributes: aptos_framework::extended_checks::get_all_attribute_names().clone(),
         language_version: None,
path: aptos-move/aptos-transactional-test-harness/src/aptos_test_harness.rs
user: login: igor-aptos
body: None is treated as default
created_at: 2025-05-08T15:48:29Z
author_association: CONTRIBUTOR
diff_hunk: @@ -320,37 +314,50 @@ static PRECOMPILED_APTOS_FRAMEWORK_V2: Lazy<PrecompiledFilesModules> = Lazy::new
         .cloned()
         .collect::<Vec<_>>();
 
-    let mut options = move_compiler_v2::Options {
-        sources: prod_sources,
+    let options = move_compiler_v2::Options {
+        sources: prod_sources.clone(),
         dependencies: vec![],
         named_address_mapping: named_address_mapping_strings.clone(),
         known_attributes: aptos_framework::extended_checks::get_all_attribute_names().clone(),
         language_version: None,
         ..move_compiler_v2::Options::default()
     };
 
-    let (_global_env, mut modules) = move_compiler_v2::run_move_compiler_to_stderr(options.clone())
+    let (_global_env, modules) = move_compiler_v2::run_move_compiler_to_stderr(options.clone())
         .expect("framework compilation succeeds");
 
-    options.sources = all_sources;
-    options.language_version = Some(LanguageVersion::latest());
+    PrecompiledFilesModules::new(prod_sources, modules)
+});
+
+static PRECOMPILED_APTOS_FRAMEWORK_V2_WITH_EXPERIMENTAL: Lazy<PrecompiledFilesModules> =
path: aptos-move/aptos-transactional-test-harness/src/aptos_test_harness.rs
user: login: igor-aptos
body: updated
created_at: 2025-05-08T19:00:39Z
author_association: CONTRIBUTOR

PR comments:
user: login: trunk-io[bot]
created_at: 2025-05-07T21:43:39Z
body: <!-- Trunk Metrics Report -->



<details>
<summary>
⏱️ 2h 29m total CI duration on this PR
</summary>

| Slowest 15 Jobs | Cumulative Duration | Recent Runs |
|---|---|---|
| [execution-performance / single-node-performance](https://github.com/aptos-labs/aptos-core/actions/runs/14916789362/job/41904316522) | 56m | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14914788505/job/41897854918)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14916789362/job/41904316522)&#8201; |
| [rust-smoke-tests](https://github.com/aptos-labs/aptos-core/actions/runs/14914788520/job/41897616859) | 37m | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14914788520/job/41897616859)&#8201; |
| [forge-framework-upgrade-test / forge](https://github.com/aptos-labs/aptos-core/actions/runs/14914788602/job/41898484893) | 15m | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14914788602/job/41898484893)&#8201; |
| [execution-performance / test-target-determinator](https://github.com/aptos-labs/aptos-core/actions/runs/14916789362/job/41904126351) | 9m | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14914788505/job/41897601875)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14916789362/job/41904126351)&#8201; |
| [test-target-determinator](https://github.com/aptos-labs/aptos-core/actions/runs/14916789345/job/41904133185) | 8m | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14914788602/job/41897613234)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14916789345/job/41904133185)&#8201; |
| [rust-cargo-deny](https://github.com/aptos-labs/aptos-core/actions/runs/14916789336/job/41904137179) | 7m | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14893846571/job/41832127752)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14914788520/job/41897616002)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14916789336/job/41904137179)&#8201; |
| [rust-doc-tests](https://github.com/aptos-labs/aptos-core/actions/runs/14914788520/job/41897602435) | 6m | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14914788520/job/41897602435)&#8201; |
| [check-dynamic-deps](https://github.com/aptos-labs/aptos-core/actions/runs/14916789337/job/41904125471) | 4m | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14893846549/job/41832111615)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14914788509/job/41897601194)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14916789337/job/41904125471)&#8201; |
| [fetch-last-released-docker-image-tag](https://github.com/aptos-labs/aptos-core/actions/runs/14916789345/job/41904133398) | 3m | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14914788602/job/41897613941)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14916789345/job/41904133398)&#8201; |
| [general-lints](https://github.com/aptos-labs/aptos-core/actions/runs/14916789336/job/41904137947) | 1m | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14893846571/job/41832128251)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14914788520/job/41897616418)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14916789336/job/41904137947)&#8201; |
| [semgrep/ci](https://github.com/aptos-labs/aptos-core/actions/runs/14916789344/job/41904125726) | 1m | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14893846542/job/41832111647)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14914788524/job/41897601290)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14916789344/job/41904125726)&#8201; |
| [file_change_determinator](https://github.com/aptos-labs/aptos-core/actions/runs/14916789336/job/41904126097) | 32s | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14893846571/job/41832112591)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14914788520/job/41897602084)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14916789336/job/41904126097)&#8201; |
| [file_change_determinator](https://github.com/aptos-labs/aptos-core/actions/runs/14916789345/job/41904132875) | 20s | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14914788602/job/41897613622)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14916789345/job/41904132875)&#8201; |
| [permission-check](https://github.com/aptos-labs/aptos-core/actions/runs/14914788528/job/41897601606) | 10s | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14893846539/job/41832111546)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14914026271/job/41895100442)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14914788528/job/41897601606)&#8201; |
| [permission-check](https://github.com/aptos-labs/aptos-core/actions/runs/14914788511/job/41897601390) | 7s | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14893846543/job/41832112159)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14914026273/job/41895100424)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14914788511/job/41897601390)&#8201; |
</details>

🚨 1 job on the last run was significantly faster/slower than expected

| Job | Duration | vs 7d avg | Delta |
|---|---|---|---|
| [rust-cargo-deny](https://github.com/aptos-labs/aptos-core/actions/runs/14916789336/job/41904137179) | 3m | [2m](https://app.trunk.io/aptoslabs/ci-analytics/1_750a2cec-81f6-4bb8-9b9c-80b3f99f60f6/1_d2e24444-88c6-4334-a2fa-a3d1b5d362b1?repo=aptos-labs/aptos-core&timeBased=true&numericAggregateFunctions=NUMERIC_AGGREGATE_FUNCTION_AVG&timeRange=7+days&conclusion=GIT_HUB_REPO_ACTIVITY_RUN_CONCLUSION_SUCCESS&utm_source=gh-comment) | [![+94%](https://raster.shields.io/badge/+94%25-A72218)](https://app.trunk.io/aptoslabs/ci-analytics/1_750a2cec-81f6-4bb8-9b9c-80b3f99f60f6/1_d2e24444-88c6-4334-a2fa-a3d1b5d362b1?repo=aptos-labs/aptos-core&timeBased=true&numericAggregateFunctions=NUMERIC_AGGREGATE_FUNCTION_AVG&timeRange=7+days&conclusion=GIT_HUB_REPO_ACTIVITY_RUN_CONCLUSION_SUCCESS&utm_source=gh-comment) |

<sub>[settings](https://app.trunk.io/aptoslabs/settings/repo?repo=aptos-labs/aptos-core&utm_source=gh-comment#ci-analytics) ⋅ [feedback](https://form.typeform.com/to/w7Iv34u2?utm_source=gh-comment) ⋅ [docs](https://docs.trunk.io/ci-analytics/pull-request-metrics?utm_source=gh-comment) ⋅ learn more about [trunk.io](https://trunk.io?utm_source=gh-comment)</sub>
author_association: NONE
user: login: izziybrown
created_at: 2025-05-08T18:01:24Z
body: Noticed your stress test results—I specialize in load testing Web3 systems.
If you need ongoing QA, my rate is $100/hour or $150 per critical bug found.

On Thu, 8 May 2025, 4:49 p.m. igor-aptos, ***@***.***> wrote:

> ***@***.**** commented on this pull request.
> ------------------------------
>
> In aptos-move/aptos-transactional-test-harness/src/aptos_test_harness.rs
> <https://github.com/aptos-labs/aptos-core/pull/16555#discussion_r2079994705>
> :
>
> > @@ -320,37 +314,50 @@ static PRECOMPILED_APTOS_FRAMEWORK_V2: Lazy<PrecompiledFilesModules> = Lazy::new
>          .cloned()
>          .collect::<Vec<_>>();
>
> -    let mut options = move_compiler_v2::Options {
> -        sources: prod_sources,
> +    let options = move_compiler_v2::Options {
> +        sources: prod_sources.clone(),
>          dependencies: vec![],
>          named_address_mapping: named_address_mapping_strings.clone(),
>          known_attributes: aptos_framework::extended_checks::get_all_attribute_names().clone(),
>          language_version: None,
>
> None is treated as default
>
> —
> Reply to this email directly, view it on GitHub
> <https://github.com/aptos-labs/aptos-core/pull/16555#discussion_r2079994705>,
> or unsubscribe
> <https://github.com/notifications/unsubscribe-auth/BP2IZ7B3FIF3FKOXSK6SGQ325N4HRAVCNFSM6AAAAAB4U6SK56VHI2DSMVQWIX3LMV43YUDVNRWFEZLROVSXG5CSMV3GSZLXHMZDQMRVGYYDMNZZGI>
> .
> You are receiving this because you are subscribed to this thread.Message
> ID: ***@***.***>
>

author_association: NONE
user: login: github-actions[bot]
created_at: 2025-05-08T20:02:29Z
body: ### Forge is running suite `realistic_env_max_load` on `3e3c49f63e051b089606e78a9382969232d59d71`
* [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-e2e-pr-16555&var-metrics_source=All&var-chain_name=forge-0&refresh=10s&from=now-15m&to=now)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-e2e-pr-16555%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-e2e-pr-16555%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16555%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16555%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22quickRange%22%3A%20%2230m%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16555&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16555&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14914788602)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-e2e -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-05-08T20:02:30Z
body: ### Forge is running suite `framework_upgrade` on `16808c72122fa780cda3de5447f7d5393dd29f72` ==> `3e3c49f63e051b089606e78a9382969232d59d71`
* [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-framework-upgrade-pr-16555&var-metrics_source=All&var-chain_name=forge-0&refresh=10s&from=now-15m&to=now)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-framework-upgrade-pr-16555%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-framework-upgrade-pr-16555%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16555%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16555%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22quickRange%22%3A%20%2230m%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16555&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16555&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14914788602)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-framework-upgrade -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-05-08T20:02:30Z
body: ### Forge is running suite `compat` on `16808c72122fa780cda3de5447f7d5393dd29f72` ==> `3e3c49f63e051b089606e78a9382969232d59d71`
* [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-compat-pr-16555&var-metrics_source=All&var-chain_name=forge-0&refresh=10s&from=now-15m&to=now)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-compat-pr-16555%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-compat-pr-16555%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16555%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16555%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22quickRange%22%3A%20%2230m%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16555&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16555&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14914788602)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-compat -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-05-08T20:13:10Z
body: ### :white_check_mark: Forge suite `compat` success on `16808c72122fa780cda3de5447f7d5393dd29f72` ==> `3e3c49f63e051b089606e78a9382969232d59d71`
```
Forge report malformed: Expecting property name enclosed in double quotes: line 20 column 1 (char 541)
'{\n  "metrics": [\n    {\n      "test_name": "compatibility::simple-validator-upgrade::liveness-check",\n      "metric": "submitted_txn",\n      "value": 298460.0\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::liveness-check",\n      "metric": "expired_txn",\n      "value": 0.0\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::liveness-check",\n      "metric": "avg_tps",\n      "value": 8992.871823651094\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::liveness-check",\n[2025-05-08T20:13:06Z INFO  aptos_forge::report] Test Ok\n      "metric": "avg_latency",\n      "value": 3786.5056389465926\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::liveness-check",\n      "metric": "p50_latency",\n      "value": 4000.0\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::liveness-check",\n      "metric": "p90_latency",\n      "value": 4600.0\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::liveness-check",\n      "metric": "p99_latency",\n      "value": 4900.0\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::single-validator-upgrade",\n      "metric": "submitted_txn",\n      "value": 89280.0\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::single-validator-upgrade",\n      "metric": "expired_txn",\n      "value": 0.0\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::single-validator-upgrade",\n      "metric": "avg_tps",\n      "value": 2321.237588686587\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::single-validator-upgrade",\n      "metric": "avg_latency",\n      "value": 14629.02447356631\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::single-validator-upgrade",\n      "metric": "p50_latency",\n      "value": 15900.0\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::single-validator-upgrade",\n      "metric": "p90_latency",\n      "value": 16900.0\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::single-validator-upgrade",\n      "metric": "p99_latency",\n      "value": 17300.0\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::half-validator-upgrade",\n      "metric": "submitted_txn",\n      "value": 99520.0\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::half-validator-upgrade",\n      "metric": "expired_txn",\n      "value": 0.0\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::half-validator-upgrade",\n      "metric": "avg_tps",\n      "value": 2666.53568103235\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::half-validator-upgrade",\n      "metric": "avg_latency",\n      "value": 12516.057325160771\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::half-validator-upgrade",\n      "metric": "p50_latency",\n      "value": 14100.0\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::half-validator-upgrade",\n      "metric": "p90_latency",\n      "value": 14700.0\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::half-validator-upgrade",\n      "metric": "p99_latency",\n      "value": 14800.0\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::rest-validator-upgrade",\n      "metric": "submitted_txn",\n      "value": 178180.0\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::rest-validator-upgrade",\n      "metric": "expired_txn",\n      "value": 0.0\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::rest-validator-upgrade",\n      "metric": "avg_tps",\n      "value": 5094.660787051723\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::rest-validator-upgrade",\n      "metric": "avg_latency",\n      "value": 6750.180991132563\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::rest-validator-upgrade",\n      "metric": "p50_latency",\n      "value": 7300.0\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::rest-validator-upgrade",\n      "metric": "p90_latency",\n      "value": 7900.0\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::rest-validator-upgrade",\n      "metric": "p99_latency",\n      "value": 7900.0\n    }\n  ],\n  "text": "Compatibility test results for 16808c72122fa780cda3de5447f7d5393dd29f72 ==> 3e3c49f63e051b089606e78a9382969232d59d71 (PR)\\n1. Check liveness of validators at old version: 16808c72122fa780cda3de5447f7d5393dd29f72\\ncompatibility::simple-validator-upgrade::liveness-check : committed: 8992.87 txn/s, latency: 3786.51 ms, (p50: 4000 ms, p70: 4200, p90: 4600 ms, p99: 4900 ms), latency samples: 298460\\n2. Upgrading first Validator to new version: 3e3c49f63e051b089606e78a9382969232d59d71\\ncompatibility::simple-validator-upgrade::single-validator-upgrade : committed: 2321.24 txn/s, latency: 14629.02 ms, (p50: 15900 ms, p70: 16500, p90: 16900 ms, p99: 17300 ms), latency samples: 89280\\n3. Upgrading rest of first batch to new version: 3e3c49f63e051b089606e78a9382969232d59d71\\ncompatibility::simple-validator-upgrade::half-validator-upgrade : committed: 2666.54 txn/s, latency: 12516.06 ms, (p50: 14100 ms, p70: 14400, p90: 14700 ms, p99: 14800 ms), latency samples: 99520\\n4. upgrading second batch to new version: 3e3c49f63e051b089606e78a9382969232d59d71\\ncompatibility::simple-validator-upgrade::rest-validator-upgrade : committed: 5094.66 txn/s, latency: 6750.18 ms, (p50: 7300 ms, p70: 7500, p90: 7900 ms, p99: 7900 ms), latency samples: 178180\\n5. check swarm health\\nCompatibility test for 16808c72122fa780cda3de5447f7d5393dd29f72 ==> 3e3c49f63e051b089606e78a9382969232d59d71 passed\\nTest Ok"\n}'
Trailing Log Lines:
2. Upgrading first Validator to new version: 3e3c49f63e051b089606e78a9382969232d59d71
compatibility::simple-validator-upgrade::single-validator-upgrade : committed: 2321.24 txn/s, latency: 14629.02 ms, (p50: 15900 ms, p70: 16500, p90: 16900 ms, p99: 17300 ms), latency samples: 89280
3. Upgrading rest of first batch to new version: 3e3c49f63e051b089606e78a9382969232d59d71
compatibility::simple-validator-upgrade::half-validator-upgrade : committed: 2666.54 txn/s, latency: 12516.06 ms, (p50: 14100 ms, p70: 14400, p90: 14700 ms, p99: 14800 ms), latency samples: 99520
4. upgrading second batch to new version: 3e3c49f63e051b089606e78a9382969232d59d71
compatibility::simple-validator-upgrade::rest-validator-upgrade : committed: 5094.66 txn/s, latency: 6750.18 ms, (p50: 7300 ms, p70: 7500, p90: 7900 ms, p99: 7900 ms), latency samples: 178180
5. check swarm health
Compatibility test for 16808c72122fa780cda3de5447f7d5393dd29f72 ==> 3e3c49f63e051b089606e78a9382969232d59d71 passed
Test Ok

=== BEGIN JUNIT ===
<?xml version="1.0" encoding="UTF-8"?>
<testsuites name="forge" tests="1" failures="0" errors="0" uuid="b3bff03a-26cc-447e-b7a5-4a832923aa86">
    <testsuite name="compat" tests="1" disabled="0" errors="0" failures="0">
        <testcase name="compatibility::simple-validator-upgrade">
        </testcase>
    </testsuite>
</testsuites>
=== END JUNIT ===
[2025-05-08T20:13:06Z INFO  aptos_forge::backend::k8s::cluster_helper] Deleting namespace forge-compat-pr-16555: Some(NamespaceStatus { conditions: None, phase: Some("Terminating") })
[2025-05-08T20:13:06Z INFO  aptos_forge::backend::k8s::cluster_helper] aptos-node resources for Forge removed in namespace: forge-compat-pr-16555

test result: ok. 1 passed; 0 failed; 0 filtered out

Debugging output:
NAME                                    READY   STATUS      RESTARTS   AGE
aptos-node-0-validator-0                1/1     Running     0          4m36s
aptos-node-1-validator-0                1/1     Running     0          6m31s
aptos-node-2-validator-0                1/1     Running     0          2m46s
aptos-node-3-validator-0                1/1     Running     0          104s
forge-testnet-deployer-5zszn            0/1     Completed   0          10m
genesis-aptos-genesis-eforge244-s28d7   0/1     Completed   0          10m

```
* [Grafana dashboard](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-compat-pr-16555&var-metrics_source=All&var-chain_name=forge-0&from=1746734557000&to=1746735188000)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-compat-pr-16555%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-compat-pr-16555%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1746734557000&end=1746735188000)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16555%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16555%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22startTime%22%3A%20%222025-05-08T20%3A02%3A37.200024%2B00%3A00%22%2C%20%22endTime%22%3A%20%222025-05-08T20%3A13%3A08.355054%2B00%3A00%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1746734557&until=1746735188&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16555&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1746734557&until=1746735188&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16555&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14914788602)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-compat -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-05-08T20:13:39Z
body: ### :white_check_mark: Forge suite `realistic_env_max_load` success on `3e3c49f63e051b089606e78a9382969232d59d71`
```
two traffics test: inner traffic : committed: 11466.45 txn/s, latency: 3144.57 ms, (p50: 2700 ms, p70: 3000, p90: 4500 ms, p99: 6600 ms), latency samples: 4359780
two traffics test : committed: 99.98 txn/s, latency: 1725.18 ms, (p50: 1400 ms, p70: 1800, p90: 2400 ms, p99: 5700 ms), latency samples: 1720
Latency breakdown for phase 0: ["MempoolToBlockCreation: max: 2.230, avg: 2.143", "ConsensusProposalToOrdered: max: 0.303, avg: 0.297", "ConsensusOrderedToCommit: max: 0.341, avg: 0.331", "ConsensusProposalToCommit: max: 0.638, avg: 0.628"]
Max non-epoch-change gap was: 0 rounds at version 0 (avg 0.00) [limit 4], 0.96s no progress at version 19418 (avg 0.20s) [limit 15].
Max epoch-change gap was: 0 rounds at version 0 (avg 0.00) [limit 4], 0.79s no progress at version 2278854 (avg 0.79s) [limit 16].
Test Ok
```
* [Grafana dashboard](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-e2e-pr-16555&var-metrics_source=All&var-chain_name=forge-0&from=1746734555000&to=1746735218000)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-e2e-pr-16555%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-e2e-pr-16555%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1746734555000&end=1746735218000)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16555%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16555%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22startTime%22%3A%20%222025-05-08T20%3A02%3A35.527636%2B00%3A00%22%2C%20%22endTime%22%3A%20%222025-05-08T20%3A13%3A38.379768%2B00%3A00%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1746734555&until=1746735218&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16555&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1746734555&until=1746735218&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16555&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14914788602)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-e2e -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-05-08T20:16:03Z
body: ### :white_check_mark: Forge suite `framework_upgrade` success on `16808c72122fa780cda3de5447f7d5393dd29f72` ==> `3e3c49f63e051b089606e78a9382969232d59d71`
```
Compatibility test results for 16808c72122fa780cda3de5447f7d5393dd29f72 ==> 3e3c49f63e051b089606e78a9382969232d59d71 (PR)
Upgrade the nodes to version: 3e3c49f63e051b089606e78a9382969232d59d71
framework_upgrade::framework-upgrade::full-framework-upgrade : committed: 1544.91 txn/s, submitted: 1548.41 txn/s, failed submission: 3.50 txn/s, expired: 3.50 txn/s, latency: 1883.00 ms, (p50: 1600 ms, p70: 2000, p90: 3000 ms, p99: 4600 ms), latency samples: 141140
framework_upgrade::framework-upgrade::full-framework-upgrade : committed: 1616.79 txn/s, submitted: 1622.55 txn/s, failed submission: 5.76 txn/s, expired: 5.76 txn/s, latency: 1916.44 ms, (p50: 1600 ms, p70: 2100, p90: 3000 ms, p99: 4900 ms), latency samples: 145880
5. check swarm health
Compatibility test for 16808c72122fa780cda3de5447f7d5393dd29f72 ==> 3e3c49f63e051b089606e78a9382969232d59d71 passed
Upgrade the remaining nodes to version: 3e3c49f63e051b089606e78a9382969232d59d71
framework_upgrade::framework-upgrade::full-framework-upgrade : committed: 1743.52 txn/s, submitted: 1748.77 txn/s, failed submission: 5.25 txn/s, expired: 5.25 txn/s, latency: 1745.26 ms, (p50: 1500 ms, p70: 1800, p90: 2800 ms, p99: 4000 ms), latency samples: 152701
Test Ok
```
* [Grafana dashboard](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-framework-upgrade-pr-16555&var-metrics_source=All&var-chain_name=forge-0&from=1746734556000&to=1746735361000)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-framework-upgrade-pr-16555%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-framework-upgrade-pr-16555%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1746734556000&end=1746735361000)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16555%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16555%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22startTime%22%3A%20%222025-05-08T20%3A02%3A36.808969%2B00%3A00%22%2C%20%22endTime%22%3A%20%222025-05-08T20%3A16%3A01.843739%2B00%3A00%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1746734556&until=1746735361&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16555&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1746734556&until=1746735361&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16555&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14914788602)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-framework-upgrade -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-05-08T22:05:26Z
body: ### Forge is running suite `realistic_env_max_load` on `50b67bf5e2d4cc0c4ac544acea48bbfda0d099af`
* [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-e2e-pr-16555&var-metrics_source=All&var-chain_name=forge-0&refresh=10s&from=now-15m&to=now)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-e2e-pr-16555%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-e2e-pr-16555%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16555%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16555%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22quickRange%22%3A%20%2230m%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16555&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16555&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14916789345)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-e2e -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-05-08T22:05:29Z
body: ### Forge is running suite `framework_upgrade` on `16808c72122fa780cda3de5447f7d5393dd29f72` ==> `50b67bf5e2d4cc0c4ac544acea48bbfda0d099af`
* [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-framework-upgrade-pr-16555&var-metrics_source=All&var-chain_name=forge-0&refresh=10s&from=now-15m&to=now)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-framework-upgrade-pr-16555%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-framework-upgrade-pr-16555%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16555%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16555%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22quickRange%22%3A%20%2230m%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16555&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16555&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14916789345)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-framework-upgrade -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-05-08T22:05:30Z
body: ### Forge is running suite `compat` on `16808c72122fa780cda3de5447f7d5393dd29f72` ==> `50b67bf5e2d4cc0c4ac544acea48bbfda0d099af`
* [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-compat-pr-16555&var-metrics_source=All&var-chain_name=forge-0&refresh=10s&from=now-15m&to=now)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-compat-pr-16555%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-compat-pr-16555%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16555%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16555%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22quickRange%22%3A%20%2230m%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16555&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16555&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14916789345)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-compat -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-05-08T22:15:57Z
body: ### :white_check_mark: Forge suite `compat` success on `16808c72122fa780cda3de5447f7d5393dd29f72` ==> `50b67bf5e2d4cc0c4ac544acea48bbfda0d099af`
```
Compatibility test results for 16808c72122fa780cda3de5447f7d5393dd29f72 ==> 50b67bf5e2d4cc0c4ac544acea48bbfda0d099af (PR)
1. Check liveness of validators at old version: 16808c72122fa780cda3de5447f7d5393dd29f72
compatibility::simple-validator-upgrade::liveness-check : committed: 9239.81 txn/s, latency: 3670.37 ms, (p50: 4000 ms, p70: 4100, p90: 4400 ms, p99: 4500 ms), latency samples: 306520
2. Upgrading first Validator to new version: 50b67bf5e2d4cc0c4ac544acea48bbfda0d099af
compatibility::simple-validator-upgrade::single-validator-upgrade : committed: 2855.81 txn/s, latency: 11657.81 ms, (p50: 13300 ms, p70: 13500, p90: 13600 ms, p99: 13900 ms), latency samples: 107220
3. Upgrading rest of first batch to new version: 50b67bf5e2d4cc0c4ac544acea48bbfda0d099af
compatibility::simple-validator-upgrade::half-validator-upgrade : committed: 2527.73 txn/s, latency: 13210.12 ms, (p50: 14800 ms, p70: 15100, p90: 15400 ms, p99: 15500 ms), latency samples: 95600
4. upgrading second batch to new version: 50b67bf5e2d4cc0c4ac544acea48bbfda0d099af
compatibility::simple-validator-upgrade::rest-validator-upgrade : committed: 5466.50 txn/s, latency: 5931.24 ms, (p50: 6500 ms, p70: 7200, p90: 7700 ms, p99: 9300 ms), latency samples: 180680
5. check swarm health
Compatibility test for 16808c72122fa780cda3de5447f7d5393dd29f72 ==> 50b67bf5e2d4cc0c4ac544acea48bbfda0d099af passed
Test Ok
```
* [Grafana dashboard](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-compat-pr-16555&var-metrics_source=All&var-chain_name=forge-0&from=1746741937000&to=1746742556000)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-compat-pr-16555%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-compat-pr-16555%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1746741937000&end=1746742556000)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16555%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16555%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22startTime%22%3A%20%222025-05-08T22%3A05%3A37.093168%2B00%3A00%22%2C%20%22endTime%22%3A%20%222025-05-08T22%3A15%3A56.007991%2B00%3A00%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1746741937&until=1746742556&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16555&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1746741937&until=1746742556&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16555&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14916789345)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-compat -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-05-08T22:18:11Z
body: ### :white_check_mark: Forge suite `realistic_env_max_load` success on `50b67bf5e2d4cc0c4ac544acea48bbfda0d099af`
```
two traffics test: inner traffic : committed: 10782.57 txn/s, submitted: 10790.31 txn/s, expired: 7.73 txn/s, latency: 3343.58 ms, (p50: 2700 ms, p70: 3400, p90: 4900 ms, p99: 7600 ms), latency samples: 4099759
two traffics test : committed: 100.00 txn/s, latency: 2294.21 ms, (p50: 1400 ms, p70: 2300, p90: 4800 ms, p99: 12300 ms), latency samples: 1820
Latency breakdown for phase 0: ["MempoolToBlockCreation: max: 2.523, avg: 2.062", "ConsensusProposalToOrdered: max: 0.324, avg: 0.305", "ConsensusOrderedToCommit: max: 0.400, avg: 0.347", "ConsensusProposalToCommit: max: 0.715, avg: 0.652"]
Max non-epoch-change gap was: 2 rounds at version 2815355 (avg 0.00) [limit 4], 3.51s no progress at version 2815355 (avg 0.21s) [limit 15].
Max epoch-change gap was: 0 rounds at version 0 (avg 0.00) [limit 4], 0.81s no progress at version 2096200 (avg 0.81s) [limit 16].
Test Ok
```
* [Grafana dashboard](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-e2e-pr-16555&var-metrics_source=All&var-chain_name=forge-0&from=1746741932000&to=1746742689000)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-e2e-pr-16555%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-e2e-pr-16555%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1746741932000&end=1746742689000)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16555%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16555%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22startTime%22%3A%20%222025-05-08T22%3A05%3A32.888733%2B00%3A00%22%2C%20%22endTime%22%3A%20%222025-05-08T22%3A18%3A09.641953%2B00%3A00%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1746741932&until=1746742689&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16555&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1746741932&until=1746742689&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16555&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14916789345)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-e2e -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-05-08T22:18:57Z
body: ### :white_check_mark: Forge suite `framework_upgrade` success on `16808c72122fa780cda3de5447f7d5393dd29f72` ==> `50b67bf5e2d4cc0c4ac544acea48bbfda0d099af`
```
Compatibility test results for 16808c72122fa780cda3de5447f7d5393dd29f72 ==> 50b67bf5e2d4cc0c4ac544acea48bbfda0d099af (PR)
Upgrade the nodes to version: 50b67bf5e2d4cc0c4ac544acea48bbfda0d099af
framework_upgrade::framework-upgrade::full-framework-upgrade : committed: 1720.49 txn/s, submitted: 1723.67 txn/s, failed submission: 3.19 txn/s, expired: 3.19 txn/s, latency: 1777.98 ms, (p50: 1600 ms, p70: 1900, p90: 2500 ms, p99: 3600 ms), latency samples: 151202
framework_upgrade::framework-upgrade::full-framework-upgrade : committed: 1633.91 txn/s, submitted: 1639.24 txn/s, failed submission: 5.33 txn/s, expired: 5.33 txn/s, latency: 1858.96 ms, (p50: 1500 ms, p70: 2000, p90: 3000 ms, p99: 3900 ms), latency samples: 147142
5. check swarm health
Compatibility test for 16808c72122fa780cda3de5447f7d5393dd29f72 ==> 50b67bf5e2d4cc0c4ac544acea48bbfda0d099af passed
Upgrade the remaining nodes to version: 50b67bf5e2d4cc0c4ac544acea48bbfda0d099af
framework_upgrade::framework-upgrade::full-framework-upgrade : committed: 1609.32 txn/s, submitted: 1615.77 txn/s, failed submission: 6.45 txn/s, expired: 6.45 txn/s, latency: 1803.73 ms, (p50: 1600 ms, p70: 2000, p90: 2600 ms, p99: 3500 ms), latency samples: 144740
Test Ok
```
* [Grafana dashboard](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-framework-upgrade-pr-16555&var-metrics_source=All&var-chain_name=forge-0&from=1746741935000&to=1746742735000)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-framework-upgrade-pr-16555%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-framework-upgrade-pr-16555%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1746741935000&end=1746742735000)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16555%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16555%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22startTime%22%3A%20%222025-05-08T22%3A05%3A35.515073%2B00%3A00%22%2C%20%22endTime%22%3A%20%222025-05-08T22%3A18%3A55.801516%2B00%3A00%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1746741935&until=1746742735&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16555&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1746741935&until=1746742735&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16555&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14916789345)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-framework-upgrade -->
author_association: CONTRIBUTOR