Repository: aptos-labs/aptos-core
Pull request #16090: [aptos-vm] Move Aptos metadata to aptos-types (1/2)
Author: login: georgemitenkov
Created: 2025-03-10T11:58:25Z | Merged: 2025-03-26T19:41:49Z

## Description

This is the 1st out of 2 PRs. The goal is to cache Aptos metadata in module cache, to avoid deserialization on every resource group access and cache it across blocks.

This PR **only** refactors code to ensure metadata is not under `framework.rs` but is defined globally in `aptos-types` so it can be used by loader, etc. without cyclic dependencies.

There are no changes to the code.

## How Has This Been Tested?
<!--
- Please ensure that the functionality introduced by this change is well tested and verified to work as expected.
- Ensure tests cover both happy and unhappy paths.
- List and link relevant tests.
-->

## Key Areas to Review
<!--
- Identify any critical parts of the code that require special attention or understanding. Explain why these parts are crucial to the functionality or architecture of the project.
- Point out any areas where complex logic has been implemented. Provide a brief explanation of the logic and your approach to make it easier for reviewers to follow.
- Highlight any areas where you are particularly concerned or unsure about the code's impact on the change. This can include potential performance or security issues, or compatibility with existing features.
-->

## Type of Change

- [x] Refactoring

## Which Components or Systems Does This Change Impact?

- [x] Move/Aptos Virtual Machine

## Checklist
- [x] I have read and followed the [CONTRIBUTING](https://github.com/aptos-labs/aptos-core/blob/main/CONTRIBUTING.md) doc
- [x] I have performed a self-review of my own code
- [x] I have commented my code, particularly in hard-to-understand areas
- [x] I identified and added all stakeholders and component owners affected by this change as reviewers
- [x] I tested both happy and unhappy path of the functionality
- [x] I have made corresponding changes to the documentation

<!-- Thank you for your contribution! -->



Diff:
diff --git a/Cargo.lock b/Cargo.lock
index 2042853835a..d8287e98482 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -570,7 +570,6 @@ dependencies = [
  "anyhow",
  "aptos-config",
  "aptos-crypto",
- "aptos-framework",
  "aptos-logger",
  "aptos-openapi",
  "aptos-resource-viewer",
@@ -1911,7 +1910,6 @@ dependencies = [
  "legacy-move-compiler",
  "libsecp256k1",
  "log",
- "lru 0.7.8",
  "merlin",
  "move-binary-format",
  "move-cli",
@@ -1942,7 +1940,6 @@ dependencies = [
  "siphasher",
  "smallvec",
  "tempfile",
- "thiserror",
  "tiny-keccak",
  "toml 0.7.8",
 ]
@@ -4465,8 +4462,10 @@ dependencies = [
  "hex",
  "itertools 0.13.0",
  "jsonwebtoken 8.3.0",
+ "lru 0.7.8",
  "move-binary-format",
  "move-core-types",
+ "move-model",
  "move-table-extension",
  "move-vm-types",
  "num-bigint 0.3.3",
diff --git a/api/types/Cargo.toml b/api/types/Cargo.toml
index 4f395c7457d..b0ac815b786 100644
--- a/api/types/Cargo.toml
+++ b/api/types/Cargo.toml
@@ -16,7 +16,6 @@ rust-version = { workspace = true }
 anyhow = { workspace = true }
 aptos-config = { workspace = true }
 aptos-crypto = { workspace = true }
-aptos-framework = { workspace = true }
 aptos-logger = { workspace = true }
 aptos-openapi = { workspace = true }
 aptos-resource-viewer = { workspace = true }
diff --git a/api/types/src/bytecode.rs b/api/types/src/bytecode.rs
index f6ed8405471..edf695577ce 100644
--- a/api/types/src/bytecode.rs
+++ b/api/types/src/bytecode.rs
@@ -9,9 +9,7 @@ use crate::{
     },
     MoveFunction, MoveStructTag, MoveType,
 };
-use aptos_framework::{
-    get_metadata_from_compiled_module, get_metadata_from_compiled_script, RuntimeModuleMetadataV1,
-};
+use aptos_types::vm::module_metadata::prelude::*;
 use aptos_vm::determine_is_view;
 use move_binary_format::{
     access::{ModuleAccess, ScriptAccess},
diff --git a/api/types/src/convert.rs b/api/types/src/convert.rs
index cc8830f02e9..61f7b8750f9 100644
--- a/api/types/src/convert.rs
+++ b/api/types/src/convert.rs
@@ -36,6 +36,7 @@ use aptos_types::{
         BlockEndInfo, BlockEpiloguePayload, EntryFunction, ExecutionStatus, Multisig,
         RawTransaction, Script, SignedTransaction, TransactionAuxiliaryData,
     },
+    vm::module_metadata::get_metadata,
     vm_status::AbortLocation,
     write_set::WriteOp,
 };
@@ -98,7 +99,7 @@ impl<'a, S: StateView> MoveConverter<'a, S> {
 
     pub fn is_resource_group(&self, tag: &StructTag) -> bool {
         if let Ok(Some(module)) = self.inner.view_module(&tag.module_id()) {
-            if let Some(md) = aptos_framework::get_metadata(&module.metadata) {
+            if let Some(md) = get_metadata(&module.metadata) {
                 if let Some(attrs) = md.struct_attributes.get(tag.name.as_ident_str().as_str()) {
                     return attrs
                         .iter()
diff --git a/aptos-move/aptos-vm/src/aptos_vm.rs b/aptos-move/aptos-vm/src/aptos_vm.rs
index cc6b4a157f6..f88dc0a5ac7 100644
--- a/aptos-move/aptos-vm/src/aptos_vm.rs
+++ b/aptos-move/aptos-vm/src/aptos_vm.rs
@@ -33,7 +33,7 @@ use aptos_block_executor::{
     txn_provider::{default::DefaultTxnProvider, TxnProvider},
 };
 use aptos_crypto::HashValue;
-use aptos_framework::{natives::code::PublishRequest, RuntimeModuleMetadataV1};
+use aptos_framework::natives::code::PublishRequest;
 use aptos_gas_algebra::{Gas, GasQuantity, NumBytes, Octa};
 use aptos_gas_meter::{AptosGasMeter, GasAlgebra};
 use aptos_gas_schedule::{
@@ -74,6 +74,11 @@ use aptos_types::{
         TransactionOutput, TransactionPayload, TransactionStatus, VMValidatorResult,
         ViewFunctionOutput, WriteSetPayload,
     },
+    vm::module_metadata::{
+        get_compilation_metadata_from_compiled_module,
+        get_compilation_metadata_from_compiled_script, get_metadata, get_metadata_v0,
+        verify_module_metadata, RuntimeModuleMetadataV1,
+    },
     vm_status::{AbortLocation, StatusCode, VMStatus},
 };
 use aptos_utils::aptos_try;
@@ -1620,7 +1625,7 @@ impl AptosVM {
                     }
                 }
             }
-            aptos_framework::verify_module_metadata(m, self.features(), self.timed_features())
+            verify_module_metadata(m, self.features(), self.timed_features())
                 .map_err(|err| Self::metadata_validation_error(&err.to_string()))?;
         }
 
@@ -1644,9 +1649,7 @@ impl AptosVM {
     fn reject_unstable_bytecode(&self, modules: &[CompiledModule]) -> VMResult<()> {
         if self.chain_id().is_mainnet() {
             for module in modules {
-                if let Some(metadata) =
-                    aptos_framework::get_compilation_metadata_from_compiled_module(module)
-                {
+                if let Some(metadata) = get_compilation_metadata_from_compiled_module(module) {
                     if metadata.unstable {
                         return Err(PartialVMError::new(StatusCode::UNSTABLE_BYTECODE_REJECTED)
                             .with_message(
@@ -1663,9 +1666,7 @@ impl AptosVM {
     /// Check whether the script can be run on mainnet based on the unstable tag in the metadata
     pub fn reject_unstable_bytecode_for_script(&self, module: &CompiledScript) -> VMResult<()> {
         if self.chain_id().is_mainnet() {
-            if let Some(metadata) =
-                aptos_framework::get_compilation_metadata_from_compiled_script(module)
-            {
+            if let Some(metadata) = get_compilation_metadata_from_compiled_script(module) {
                 if metadata.unstable {
                     return Err(PartialVMError::new(StatusCode::UNSTABLE_BYTECODE_REJECTED)
                         .with_message("script marked unstable cannot be run on mainnet".to_string())
@@ -2381,10 +2382,13 @@ impl AptosVM {
         module_storage: &impl AptosModuleStorage,
         module_id: &ModuleId,
     ) -> Option<Arc<RuntimeModuleMetadataV1>> {
+        let metadata = module_storage
+            .fetch_module_metadata(module_id.address(), module_id.name())
+            .ok()??;
         if self.features().is_enabled(FeatureFlag::VM_BINARY_FORMAT_V6) {
-            aptos_framework::get_vm_metadata(module_storage, module_id)
+            get_metadata(&metadata)
         } else {
-            aptos_framework::get_vm_metadata_v0(module_storage, module_id)
+            get_metadata_v0(&metadata)
         }
     }
 
diff --git a/aptos-move/aptos-vm/src/data_cache.rs b/aptos-move/aptos-vm/src/data_cache.rs
index c560b1f794c..bd737863198 100644
--- a/aptos-move/aptos-vm/src/data_cache.rs
+++ b/aptos-move/aptos-vm/src/data_cache.rs
@@ -23,6 +23,7 @@ use aptos_types::{
         state_value::{StateValue, StateValueMetadata},
         StateView, StateViewId,
     },
+    vm::module_metadata::get_metadata,
 };
 use aptos_vm_environment::gas::get_gas_feature_version;
 use aptos_vm_types::{
@@ -51,7 +52,7 @@ pub fn get_resource_group_member_from_metadata(
     struct_tag: &StructTag,
     metadata: &[Metadata],
 ) -> Option<StructTag> {
-    let metadata = aptos_framework::get_metadata(metadata)?;
+    let metadata = get_metadata(metadata)?;
     metadata
         .struct_attributes
         .get(struct_tag.name.as_ident_str().as_str())?
diff --git a/aptos-move/aptos-vm/src/verifier/event_validation.rs b/aptos-move/aptos-vm/src/verifier/event_validation.rs
index b7101da5ddf..b628f361aa1 100644
--- a/aptos-move/aptos-vm/src/verifier/event_validation.rs
+++ b/aptos-move/aptos-vm/src/verifier/event_validation.rs
@@ -1,7 +1,9 @@
 // Copyright © Aptos Foundation
 // SPDX-License-Identifier: Apache-2.0
 
-use aptos_framework::RuntimeModuleMetadataV1;
+use aptos_types::vm::module_metadata::{
+    get_metadata_from_compiled_module, RuntimeModuleMetadataV1,
+};
 use aptos_vm_types::module_and_script_storage::module_storage::AptosModuleStorage;
 use move_binary_format::{
     access::{ModuleAccess, ScriptAccess},
@@ -39,7 +41,7 @@ pub(crate) fn validate_module_events(
 ) -> VMResult<()> {
     for module in modules {
         let mut new_event_structs =
-            if let Some(metadata) = aptos_framework::get_metadata_from_compiled_module(module) {
+            if let Some(metadata) = get_metadata_from_compiled_module(module) {
                 extract_event_metadata(&metadata)?
             } else {
                 HashSet::new()
@@ -123,7 +125,7 @@ pub(crate) fn extract_event_metadata_from_module(
     // TODO(loader_v2): We can optimize metadata calls as well.
     let metadata = module_storage
         .fetch_deserialized_module(module_id.address(), module_id.name())?
-        .map(|module| aptos_framework::get_metadata_from_compiled_module(module.as_ref()));
+        .map(|module| get_metadata_from_compiled_module(module.as_ref()));
     if let Some(Some(metadata)) = metadata {
         extract_event_metadata(&metadata)
     } else {
diff --git a/aptos-move/aptos-vm/src/verifier/randomness.rs b/aptos-move/aptos-vm/src/verifier/randomness.rs
index 3128df74cf9..d2d7f188d97 100644
--- a/aptos-move/aptos-vm/src/verifier/randomness.rs
+++ b/aptos-move/aptos-vm/src/verifier/randomness.rs
@@ -1,8 +1,12 @@
 // Copyright © Aptos Foundation
 // SPDX-License-Identifier: Apache-2.0
 
-use aptos_framework::{KnownAttribute, RandomnessAnnotation};
-use aptos_types::transaction::EntryFunction;
+use aptos_types::{
+    transaction::EntryFunction,
+    vm::module_metadata::{
+        get_metadata_from_compiled_module, KnownAttribute, RandomnessAnnotation,
+    },
+};
 use aptos_vm_types::module_and_script_storage::module_storage::AptosModuleStorage;
 use move_binary_format::errors::VMResult;
 
@@ -16,7 +20,7 @@ pub(crate) fn get_randomness_annotation(
         entry_fn.module().name(),
     )?;
 
-    let metadata = aptos_framework::get_metadata_from_compiled_module(&module);
+    let metadata = get_metadata_from_compiled_module(&module);
     if let Some(metadata) = metadata {
         let maybe_annotation = metadata
             .fun_attributes
diff --git a/aptos-move/aptos-vm/src/verifier/resource_groups.rs b/aptos-move/aptos-vm/src/verifier/resource_groups.rs
index 3b4faefd79e..f6f521eeb76 100644
--- a/aptos-move/aptos-vm/src/verifier/resource_groups.rs
+++ b/aptos-move/aptos-vm/src/verifier/resource_groups.rs
@@ -1,7 +1,9 @@
 // Copyright © Aptos Foundation
 // SPDX-License-Identifier: Apache-2.0
 
-use aptos_framework::{ResourceGroupScope, RuntimeModuleMetadataV1};
+use aptos_types::vm::module_metadata::{
+    get_metadata_from_compiled_module, ResourceGroupScope, RuntimeModuleMetadataV1,
+};
 use aptos_vm_types::module_and_script_storage::module_storage::AptosModuleStorage;
 use move_binary_format::{
     access::ModuleAccess,
@@ -84,7 +86,7 @@ pub(crate) fn validate_module_and_extract_new_entries(
     BTreeMap<String, StructTag>,
 )> {
     let (new_groups, mut new_members) =
-        if let Some(metadata) = aptos_framework::get_metadata_from_compiled_module(module) {
+        if let Some(metadata) = get_metadata_from_compiled_module(module) {
             extract_resource_group_metadata(&metadata)?
         } else {
             (BTreeMap::new(), BTreeMap::new())
@@ -153,10 +155,7 @@ pub(crate) fn extract_resource_group_metadata_from_module(
     let module =
         module_storage.fetch_existing_deserialized_module(module_id.address(), module_id.name());
     let (metadata, module) = if let Ok(module) = module {
-        (
-            aptos_framework::get_metadata_from_compiled_module(&module),
-            module,
-        )
+        (get_metadata_from_compiled_module(&module), module)
     } else {
         // Maintaining backwards compatibility with no validation of deserialization.
         return Ok((BTreeMap::new(), BTreeMap::new(), BTreeSet::new()));
diff --git a/aptos-move/aptos-vm/src/verifier/view_function.rs b/aptos-move/aptos-vm/src/verifier/view_function.rs
index 3e7a51385be..5cf151d8469 100644
--- a/aptos-move/aptos-vm/src/verifier/view_function.rs
+++ b/aptos-move/aptos-vm/src/verifier/view_function.rs
@@ -5,7 +5,7 @@ use crate::{
     move_vm_ext::{AptosMoveResolver, SessionExt},
     verifier::{transaction_arg_validation, transaction_arg_validation::get_allowed_structs},
 };
-use aptos_framework::RuntimeModuleMetadataV1;
+use aptos_types::vm::module_metadata::RuntimeModuleMetadataV1;
 use aptos_vm_types::module_and_script_storage::module_storage::AptosModuleStorage;
 use move_binary_format::errors::{PartialVMError, PartialVMResult};
 use move_core_types::{identifier::IdentStr, vm_status::StatusCode};
diff --git a/aptos-move/e2e-move-tests/src/tests/attributes.rs b/aptos-move/e2e-move-tests/src/tests/attributes.rs
index 13d58501efa..c9ecb1fae65 100644
--- a/aptos-move/e2e-move-tests/src/tests/attributes.rs
+++ b/aptos-move/e2e-move-tests/src/tests/attributes.rs
@@ -5,7 +5,11 @@ use crate::{assert_success, assert_vm_status, build_package, MoveHarness};
 use aptos_cached_packages::aptos_stdlib;
 use aptos_framework::BuildOptions;
 use aptos_package_builder::PackageBuilder;
-use aptos_types::{account_address::AccountAddress, on_chain_config::FeatureFlag};
+use aptos_types::{
+    account_address::AccountAddress,
+    on_chain_config::FeatureFlag,
+    vm::module_metadata::{KnownAttribute, RuntimeModuleMetadataV1, APTOS_METADATA_KEY_V1},
+};
 use move_binary_format::CompiledModule;
 use move_core_types::{metadata::Metadata, vm_status::StatusCode};
 use serde::Serialize;
@@ -154,7 +158,7 @@ fn test_bad_fun_attribute_in_compiled_module() {
     assert!(code.len() == 1);
     let mut compiled_module = CompiledModule::deserialize(&code[0]).unwrap();
 
-    let mut value = aptos_framework::RuntimeModuleMetadataV1 {
+    let mut value = RuntimeModuleMetadataV1 {
         error_map: BTreeMap::new(),
         struct_attributes: BTreeMap::new(),
         fun_attributes: BTreeMap::new(),
@@ -164,14 +168,13 @@ fn test_bad_fun_attribute_in_compiled_module() {
         args: vec![],
     })
     .unwrap();
-    let known_attribute =
-        bcs::from_bytes::<aptos_framework::KnownAttribute>(&fake_attribute).unwrap();
+    let known_attribute = bcs::from_bytes::<KnownAttribute>(&fake_attribute).unwrap();
     value
         .fun_attributes
         .insert("view".to_string(), vec![known_attribute]);
 
     let metadata = Metadata {
-        key: aptos_framework::APTOS_METADATA_KEY_V1.to_vec(),
+        key: APTOS_METADATA_KEY_V1.to_vec(),
         value: bcs::to_bytes(&value).unwrap(),
     };
 
@@ -302,7 +305,7 @@ fn build_package_and_insert_attribute(
     // There should only be one module
     assert!(code.len() == 1);
     let mut compiled_module = CompiledModule::deserialize(&code[0]).unwrap();
-    let mut value = aptos_framework::RuntimeModuleMetadataV1 {
+    let mut value = RuntimeModuleMetadataV1 {
         error_map: BTreeMap::new(),
         struct_attributes: BTreeMap::new(),
         fun_attributes: BTreeMap::new(),
@@ -324,7 +327,7 @@ fn build_package_and_insert_attribute(
     }
 
     let metadata = Metadata {
-        key: aptos_framework::APTOS_METADATA_KEY_V1.to_vec(),
+        key: APTOS_METADATA_KEY_V1.to_vec(),
         value: bcs::to_bytes(&value).unwrap(),
     };
 
diff --git a/aptos-move/e2e-move-tests/src/tests/metadata.rs b/aptos-move/e2e-move-tests/src/tests/metadata.rs
index 6a3580aa9fa..8abe388823c 100644
--- a/aptos-move/e2e-move-tests/src/tests/metadata.rs
+++ b/aptos-move/e2e-move-tests/src/tests/metadata.rs
@@ -3,15 +3,15 @@
 
 use crate::{assert_success, assert_vm_status, build_package, MoveHarness};
 use aptos_cached_packages::aptos_stdlib;
-use aptos_framework::{
-    BuildOptions, RuntimeModuleMetadata, RuntimeModuleMetadataV1, APTOS_METADATA_KEY,
-    APTOS_METADATA_KEY_V1,
-};
+use aptos_framework::BuildOptions;
 use aptos_package_builder::PackageBuilder;
 use aptos_types::{
     chain_id::ChainId,
     on_chain_config::{FeatureFlag, OnChainConfig},
     transaction::{Script, TransactionPayload, TransactionStatus},
+    vm::module_metadata::{
+        RuntimeModuleMetadata, RuntimeModuleMetadataV1, APTOS_METADATA_KEY, APTOS_METADATA_KEY_V1,
+    },
 };
 use move_binary_format::CompiledModule;
 use move_core_types::{
diff --git a/aptos-move/framework/Cargo.toml b/aptos-move/framework/Cargo.toml
index fcf35cab4b4..f8088c64eb2 100644
--- a/aptos-move/framework/Cargo.toml
+++ b/aptos-move/framework/Cargo.toml
@@ -47,7 +47,6 @@ itertools = { workspace = true }
 legacy-move-compiler = { workspace = true }
 libsecp256k1 = { workspace = true }
 log = { workspace = true }
-lru = { workspace = true }
 merlin = { workspace = true }
 move-binary-format = { workspace = true }
 move-cli = { workspace = true }
@@ -77,7 +76,6 @@ sha3 = { workspace = true }
 siphasher = { workspace = true }
 smallvec = { workspace = true }
 tempfile = { workspace = true }
-thiserror = { workspace = true }
 tiny-keccak = { workspace = true }
 toml = { workspace = true }
 
diff --git a/aptos-move/framework/src/built_package.rs b/aptos-move/framework/src/built_package.rs
index ca443e3aa4e..87338afd0f0 100644
--- a/aptos-move/framework/src/built_package.rs
+++ b/aptos-move/framework/src/built_package.rs
@@ -5,11 +5,17 @@ use crate::{
     docgen::DocgenOptions,
     extended_checks,
     natives::code::{ModuleMetadata, MoveOption, PackageDep, PackageMetadata, UpgradePolicy},
-    zip_metadata, zip_metadata_str, RuntimeModuleMetadataV1, APTOS_METADATA_KEY,
-    APTOS_METADATA_KEY_V1, METADATA_V1_MIN_FILE_FORMAT_VERSION,
+    zip_metadata, zip_metadata_str,
 };
 use anyhow::bail;
-use aptos_types::{account_address::AccountAddress, transaction::EntryABI};
+use aptos_types::{
+    account_address::AccountAddress,
+    transaction::EntryABI,
+    vm::module_metadata::{
+        RuntimeModuleMetadataV1, APTOS_METADATA_KEY, APTOS_METADATA_KEY_V1,
+        METADATA_V1_MIN_FILE_FORMAT_VERSION,
+    },
+};
 use clap::Parser;
 use codespan_reporting::{
     diagnostic::Severity,
diff --git a/aptos-move/framework/src/extended_checks.rs b/aptos-move/framework/src/extended_checks.rs
index 920a5d6cf0b..6769b3f5bdb 100644
--- a/aptos-move/framework/src/extended_checks.rs
+++ b/aptos-move/framework/src/extended_checks.rs
@@ -1,7 +1,9 @@
 // Copyright © Aptos Foundation
 // SPDX-License-Identifier: Apache-2.0
 
-use crate::{KnownAttribute, RandomnessAnnotation, RuntimeModuleMetadataV1};
+use aptos_types::vm::module_metadata::{
+    KnownAttribute, RandomnessAnnotation, ResourceGroupScope, RuntimeModuleMetadataV1,
+};
 use legacy_move_compiler::shared::known_attributes;
 use move_binary_format::file_format::Visibility;
 use move_cli::base::test_validation;
@@ -31,9 +33,7 @@ use once_cell::sync::Lazy;
 use std::{
     collections::{BTreeMap, BTreeSet},
     rc::Rc,
-    str::FromStr,
 };
-use thiserror::Error;
 
 const ALLOW_UNSAFE_RANDOMNESS_ATTRIBUTE: &str = "lint::allow_unsafe_randomness";
 const FMT_SKIP_ATTRIBUTE: &str = "fmt::skip";
@@ -910,68 +910,3 @@ impl<'a> ExtendedChecker<'a> {
             == AccountAddress::ONE
     }
 }
-
-// ----------------------------------------------------------------------------------
-// Resource Group Container Scope
-
-#[derive(Debug, Eq, PartialEq)]
-pub enum ResourceGroupScope {
-    Global,
-    Address,
-    Module,
-}
-
-impl ResourceGroupScope {
-    pub fn is_less_strict(&self, other: &ResourceGroupScope) -> bool {
-        match self {
-            ResourceGroupScope::Global => other != self,
-            ResourceGroupScope::Address => other == &ResourceGroupScope::Module,
-            ResourceGroupScope::Module => false,
-        }
-    }
-
-    pub fn are_equal_envs(&self, resource: &StructEnv, group: &StructEnv) -> bool {
-        match self {
-            ResourceGroupScope::Global => true,
-            ResourceGroupScope::Address => {
-                resource.module_env.get_name().addr() == group.module_env.get_name().addr()
-            },
-            ResourceGroupScope::Module => {
-                resource.module_env.get_name() == group.module_env.get_name()
-            },
-        }
-    }
-
-    pub fn are_equal_module_ids(&self, resource: &ModuleId, group: &ModuleId) -> bool {
-        match self {
-            ResourceGroupScope::Global => true,
-            ResourceGroupScope::Address => resource.address() == group.address(),
-            ResourceGroupScope::Module => resource == group,
-        }
-    }
-
-    pub fn as_str(&self) -> &'static str {
-        match self {
-            ResourceGroupScope::Global => "global",
-            ResourceGroupScope::Address => "address",
-            ResourceGroupScope::Module => "module_",
-        }
-    }
-}
-
-impl FromStr for ResourceGroupScope {
-    type Err = ResourceGroupScopeError;
-
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        match s {
-            "global" => Ok(ResourceGroupScope::Global),
-            "address" => Ok(ResourceGroupScope::Address),
-            "module_" => Ok(ResourceGroupScope::Module),
-            _ => Err(ResourceGroupScopeError(s.to_string())),
-        }
-    }
-}
-
-#[derive(Debug, Error)]
-#[error("Invalid resource group scope: {0}")]
-pub struct ResourceGroupScopeError(String);
diff --git a/aptos-move/framework/src/lib.rs b/aptos-move/framework/src/lib.rs
index 5103790707c..69b17dd30ee 100644
--- a/aptos-move/framework/src/lib.rs
+++ b/aptos-move/framework/src/lib.rs
@@ -12,16 +12,12 @@ use std::io::{Read, Write};
 mod built_package;
 pub use built_package::*;
 
-mod module_metadata;
-pub use module_metadata::*;
-
 pub mod natives;
 mod release_builder;
 pub use release_builder::*;
+pub mod chunked_publish;
 pub mod docgen;
 pub mod extended_checks;
-pub use extended_checks::ResourceGroupScope;
-pub mod chunked_publish;
 pub mod prover;
 mod release_bundle;
 mod released_framework;
diff --git a/crates/aptos/src/move_tool/bytecode.rs b/crates/aptos/src/move_tool/bytecode.rs
index 0f6a466ec52..541785e2b7b 100644
--- a/crates/aptos/src/move_tool/bytecode.rs
+++ b/crates/aptos/src/move_tool/bytecode.rs
@@ -12,10 +12,7 @@ use crate::{
     update::get_revela_path,
 };
 use anyhow::Context;
-use aptos_framework::{
-    get_compilation_metadata_from_compiled_module, get_compilation_metadata_from_compiled_script,
-    get_metadata_from_compiled_module, get_metadata_from_compiled_script, RuntimeModuleMetadataV1,
-};
+use aptos_types::vm::module_metadata::prelude::*;
 use async_trait::async_trait;
 use clap::{Args, Parser};
 use itertools::Itertools;
diff --git a/crates/transaction-generator-lib/src/publishing/publish_util.rs b/crates/transaction-generator-lib/src/publishing/publish_util.rs
index c7d8dfc17ab..cbd55e48bd7 100644
--- a/crates/transaction-generator-lib/src/publishing/publish_util.rs
+++ b/crates/transaction-generator-lib/src/publishing/publish_util.rs
@@ -2,9 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use super::entry_point_trait::PreBuiltPackages;
-use aptos_framework::{
-    natives::code::PackageMetadata, KnownAttribute, APTOS_METADATA_KEY, APTOS_METADATA_KEY_V1,
-};
+use aptos_framework::natives::code::PackageMetadata;
 use aptos_sdk::{
     bcs,
     move_types::{identifier::Identifier, language_storage::ModuleId},
@@ -12,6 +10,10 @@ use aptos_sdk::{
     types::{
         account_address::AccountAddress,
         transaction::{Script, TransactionPayload},
+        vm::module_metadata::{
+            get_metadata_from_compiled_module, KnownAttribute, APTOS_METADATA_KEY,
+            APTOS_METADATA_KEY_V1,
+        },
     },
 };
 use move_binary_format::{
@@ -274,8 +276,7 @@ fn update(
                 }
             }
         }
-        if let Some(mut metadata) = aptos_framework::get_metadata_from_compiled_module(&new_module)
-        {
+        if let Some(mut metadata) = get_metadata_from_compiled_module(&new_module) {
             metadata
                 .struct_attributes
                 .iter_mut()
diff --git a/types/Cargo.toml b/types/Cargo.toml
index 8626ec2895e..8b3c28e82cc 100644
--- a/types/Cargo.toml
+++ b/types/Cargo.toml
@@ -40,8 +40,10 @@ hashbrown = { workspace = true }
 hex = { workspace = true }
 itertools = { workspace = true }
 jsonwebtoken = { workspace = true }
+lru = { workspace = true }
 move-binary-format = { workspace = true }
 move-core-types = { workspace = true }
+move-model = { workspace = true }
 move-table-extension = { workspace = true }
 move-vm-types = { workspace = true }
 num-bigint = { workspace = true }
diff --git a/types/src/vm/mod.rs b/types/src/vm/mod.rs
index 2515414ab40..c941aa5f96b 100644
--- a/types/src/vm/mod.rs
+++ b/types/src/vm/mod.rs
@@ -1,4 +1,5 @@
 // Copyright © Aptos Foundation
 // SPDX-License-Identifier: Apache-2.0
 
+pub mod module_metadata;
 pub mod modules;
diff --git a/aptos-move/framework/src/module_metadata.rs b/types/src/vm/module_metadata.rs
similarity index 83%
rename from aptos-move/framework/src/module_metadata.rs
rename to types/src/vm/module_metadata.rs
index 14522ac48bb..7ff0dcfb6d7 100644
--- a/aptos-move/framework/src/module_metadata.rs
+++ b/types/src/vm/module_metadata.rs
@@ -1,12 +1,10 @@
 // Copyright © Aptos Foundation
 // SPDX-License-Identifier: Apache-2.0
 
-use crate::extended_checks::ResourceGroupScope;
-use aptos_types::{
+use crate::{
     on_chain_config::{FeatureFlag, Features, TimedFeatureFlag, TimedFeatures},
     transaction::AbortInfo,
 };
-use aptos_vm_types::module_and_script_storage::module_storage::AptosModuleStorage;
 use lru::LruCache;
 use move_binary_format::{
     access::ModuleAccess,
@@ -23,11 +21,22 @@ use move_core_types::{
     language_storage::{ModuleId, StructTag},
     metadata::Metadata,
 };
-use move_model::metadata::{CompilationMetadata, COMPILATION_METADATA_KEY};
+use move_model::{
+    metadata::{CompilationMetadata, COMPILATION_METADATA_KEY},
+    model::StructEnv,
+};
 use serde::{Deserialize, Serialize};
-use std::{cell::RefCell, collections::BTreeMap, env, sync::Arc};
+use std::{cell::RefCell, collections::BTreeMap, env, str::FromStr, sync::Arc};
 use thiserror::Error;
 
+pub mod prelude {
+    pub use crate::vm::module_metadata::{
+        get_compilation_metadata_from_compiled_module,
+        get_compilation_metadata_from_compiled_script, get_metadata_from_compiled_module,
+        get_metadata_from_compiled_script, RuntimeModuleMetadataV1,
+    };
+}
+
 /// The minimal file format version from which the V1 metadata is supported
 pub const METADATA_V1_MIN_FILE_FORMAT_VERSION: u32 = 6;
 
@@ -227,30 +236,8 @@ pub fn get_metadata_v0(md: &[Metadata]) -> Option<Arc<RuntimeModuleMetadataV1>>
     }
 }
 
-/// Extract metadata from the VM, upgrading V0 to V1 representation as needed
-pub fn get_vm_metadata(
-    module_storage: &impl AptosModuleStorage,
-    module_id: &ModuleId,
-) -> Option<Arc<RuntimeModuleMetadataV1>> {
-    let metadata = module_storage
-        .fetch_module_metadata(module_id.address(), module_id.name())
-        .ok()??;
-    get_metadata(&metadata)
-}
-
-/// Extract metadata from the VM, legacy V0 format upgraded to V1
-pub fn get_vm_metadata_v0(
-    module_storage: &impl AptosModuleStorage,
-    module_id: &ModuleId,
-) -> Option<Arc<RuntimeModuleMetadataV1>> {
-    let metadata = module_storage
-        .fetch_module_metadata(module_id.address(), module_id.name())
-        .ok()??;
-    get_metadata_v0(&metadata)
-}
-
 /// Check if the metadata has unknown key/data types
-pub fn check_metadata_format(
+fn check_metadata_format(
     module: &CompiledModule,
     features: &Features,
 ) -> Result<(), MalformedError> {
@@ -740,3 +727,117 @@ impl RandomnessAnnotation {
         Self { max_gas }
     }
 }
+
+#[derive(Debug, Eq, PartialEq)]
+pub enum ResourceGroupScope {
+    Global,
+    Address,
+    Module,
+}
+
+impl ResourceGroupScope {
+    pub fn is_less_strict(&self, other: &ResourceGroupScope) -> bool {
+        match self {
+            ResourceGroupScope::Global => other != self,
+            ResourceGroupScope::Address => other == &ResourceGroupScope::Module,
+            ResourceGroupScope::Module => false,
+        }
+    }
+
+    pub fn are_equal_envs(&self, resource: &StructEnv, group: &StructEnv) -> bool {
+        match self {
+            ResourceGroupScope::Global => true,
+            ResourceGroupScope::Address => {
+                resource.module_env.get_name().addr() == group.module_env.get_name().addr()
+            },
+            ResourceGroupScope::Module => {
+                resource.module_env.get_name() == group.module_env.get_name()
+            },
+        }
+    }
+
+    pub fn are_equal_module_ids(&self, resource: &ModuleId, group: &ModuleId) -> bool {
+        match self {
+            ResourceGroupScope::Global => true,
+            ResourceGroupScope::Address => resource.address() == group.address(),
+            ResourceGroupScope::Module => resource == group,
+        }
+    }
+
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            ResourceGroupScope::Global => "global",
+            ResourceGroupScope::Address => "address",
+            ResourceGroupScope::Module => "module_",
+        }
+    }
+}
+
+impl FromStr for ResourceGroupScope {
+    type Err = ResourceGroupScopeError;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            "global" => Ok(ResourceGroupScope::Global),
+            "address" => Ok(ResourceGroupScope::Address),
+            "module_" => Ok(ResourceGroupScope::Module),
+            _ => Err(ResourceGroupScopeError(s.to_string())),
+        }
+    }
+}
+
+#[derive(Debug, Error)]
+#[error("Invalid resource group scope: {0}")]
+pub struct ResourceGroupScopeError(String);
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_is_less_strict() {
+        let less_strict = [
+            (ResourceGroupScope::Global, ResourceGroupScope::Address),
+            (ResourceGroupScope::Global, ResourceGroupScope::Module),
+            (ResourceGroupScope::Address, ResourceGroupScope::Module),
+        ];
+        for (scope, other_scope) in less_strict {
+            assert!(scope.is_less_strict(&other_scope));
+        }
+
+        let more_or_as_strict = [
+            (ResourceGroupScope::Global, ResourceGroupScope::Global),
+            (ResourceGroupScope::Address, ResourceGroupScope::Global),
+            (ResourceGroupScope::Address, ResourceGroupScope::Address),
+            (ResourceGroupScope::Module, ResourceGroupScope::Global),
+            (ResourceGroupScope::Module, ResourceGroupScope::Address),
+            (ResourceGroupScope::Module, ResourceGroupScope::Module),
+        ];
+        for (scope, other_scope) in more_or_as_strict {
+            assert!(!scope.is_less_strict(&other_scope));
+        }
+    }
+
+    #[test]
+    fn test_are_equal_module_ids() {
+        let id = |s: &str| -> ModuleId { ModuleId::from_str(s).unwrap() };
+
+        let global_scope = ResourceGroupScope::Global;
+        assert!(global_scope.are_equal_module_ids(&id("0x1::foo"), &id("0x1::foo")));
+        assert!(global_scope.are_equal_module_ids(&id("0x1::foo"), &id("0x1::bar")));
+        assert!(global_scope.are_equal_module_ids(&id("0x1::foo"), &id("0x2::foo")));
+        assert!(global_scope.are_equal_module_ids(&id("0x1::foo"), &id("0x2::bar")));
+
+        let address_scope = ResourceGroupScope::Address;
+        assert!(address_scope.are_equal_module_ids(&id("0x1::foo"), &id("0x1::foo")));
+        assert!(address_scope.are_equal_module_ids(&id("0x1::foo"), &id("0x1::bar")));
+        assert!(!address_scope.are_equal_module_ids(&id("0x1::foo"), &id("0x2::foo")));
+        assert!(!address_scope.are_equal_module_ids(&id("0x1::foo"), &id("0x2::bar")));
+
+        let module_scope = ResourceGroupScope::Module;
+        assert!(module_scope.are_equal_module_ids(&id("0x1::foo"), &id("0x1::foo")));
+        assert!(!module_scope.are_equal_module_ids(&id("0x1::foo"), &id("0x1::bar")));
+        assert!(!module_scope.are_equal_module_ids(&id("0x1::foo"), &id("0x2::foo")));
+        assert!(!module_scope.are_equal_module_ids(&id("0x1::foo"), &id("0x2::bar")));
+    }
+}


Reviews:
user: login: gelash
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-10T18:42:05Z
user: login: zekun000
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-12T23:05:33Z
user: login: wrwg
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-14T14:10:17Z
user: login: georgemitenkov
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-17T19:04:26Z

Review comments:
diff_hunk: @@ -77,6 +74,11 @@ use aptos_types::{
         TransactionOutput, TransactionPayload, TransactionStatus, VMValidatorResult,
         ViewFunctionOutput, WriteSetPayload,
     },
+    vm::module_metadata::{
+        get_compilation_metadata_from_compiled_module,
path: aptos-move/aptos-vm/src/aptos_vm.rs
user: login: gelash
body: does it make sense to use the prelude for these?
created_at: 2025-03-10T18:39:06Z
author_association: CONTRIBUTOR
diff_hunk: @@ -740,3 +719,65 @@ impl RandomnessAnnotation {
         Self { max_gas }
     }
 }
+
+#[derive(Debug, Eq, PartialEq)]
+pub enum ResourceGroupScope {
+    Global,
+    Address,
+    Module,
+}
+
+impl ResourceGroupScope {
+    pub fn is_less_strict(&self, other: &ResourceGroupScope) -> bool {
path: types/src/vm/module_metadata.rs
user: login: gelash
body: let's add one unit test for this and for are_equal_module_ids, also confirming that the logic makes sense?
created_at: 2025-03-10T18:42:03Z
author_association: CONTRIBUTOR
diff_hunk: @@ -77,6 +74,11 @@ use aptos_types::{
         TransactionOutput, TransactionPayload, TransactionStatus, VMValidatorResult,
         ViewFunctionOutput, WriteSetPayload,
     },
+    vm::module_metadata::{
+        get_compilation_metadata_from_compiled_module,
path: aptos-move/aptos-vm/src/aptos_vm.rs
user: login: georgemitenkov
body: Added a prelude for these, excluding `get_metadata`, `get_metadata_v0`, and `verify_module_metadata` as these are VM only (+ in 1 place in indexer but for no good reason)
created_at: 2025-03-17T19:04:26Z
author_association: CONTRIBUTOR

PR comments:
user: login: trunk-io[bot]
created_at: 2025-03-10T11:58:28Z
body: <!-- Trunk Metrics Report -->



<details>
<summary>
⏱️ 33m total CI duration on this PR
</summary>

| Job | Cumulative Duration | Recent Runs |
|---|---|---|
| [check-dynamic-deps](https://github.com/aptos-labs/aptos-core/actions/runs/13871674531/job/38819075102) | 17m | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13841315273/job/38729243910)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13854774354/job/38769084522)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13854774385/job/38769084490)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13871674617/job/38819075609)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13871674531/job/38819075102)&#8201;(+4 more) |
| [rust-cargo-deny](https://github.com/aptos-labs/aptos-core/actions/runs/13871674610/job/38819083600) | 8m | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13764063410/job/38486396862)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13764373481/job/38487402279)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13841315420/job/38729303999)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13854774405/job/38769112872)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13871674610/job/38819083600)&#8201; |
| [semgrep/ci](https://github.com/aptos-labs/aptos-core/actions/runs/13871674528/job/38819074962) | 3m | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13841315441/job/38729246433)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13854774351/job/38769085504)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13854774383/job/38769085098)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13871674611/job/38819075532)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13871674528/job/38819074962)&#8201;(+4 more) |
| [general-lints](https://github.com/aptos-labs/aptos-core/actions/runs/13871674610/job/38819083750) | 2m | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13764063410/job/38486397864)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13764373481/job/38487401495)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13841315420/job/38729304958)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13854774405/job/38769112546)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13871674610/job/38819083750)&#8201; |
| [file_change_determinator](https://github.com/aptos-labs/aptos-core/actions/runs/13871674610/job/38819079917) | 54s | [⬜](https://github.com/aptos-labs/aptos-core/actions/runs/13841315271/job/38729248958)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13841315420/job/38729274116)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13854774405/job/38769091249)&#8201;[⬜](https://github.com/aptos-labs/aptos-core/actions/runs/13871674527/job/38819076083)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13871674610/job/38819079917)&#8201;(+3 more) |
| [permission-check](https://github.com/aptos-labs/aptos-core/actions/runs/13871674448/job/38819074748) | 27s | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13841315197/job/38729242954)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13854774121/job/38769080369)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13854774234/job/38769082781)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13871674391/job/38819074530)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13871674448/job/38819074748)&#8201;(+4 more) |
| [permission-check](https://github.com/aptos-labs/aptos-core/actions/runs/13871674447/job/38819074740) | 20s | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13841315194/job/38729242950)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13854774119/job/38769080486)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13854774182/job/38769081647)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13871674397/job/38819074541)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13871674447/job/38819074740)&#8201;(+4 more) |
| [check-branch-prefix](https://github.com/aptos-labs/aptos-core/actions/runs/13764062844/job/38486379711) | 1s | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13764062844/job/38486379711)&#8201; |
</details>

<sub>[settings](https://app.trunk.io/aptoslabs/settings/repo?repo=aptos-labs/aptos-core&utm_source=gh-comment#ci-analytics) ⋅ [feedback](https://form.typeform.com/to/w7Iv34u2?utm_source=gh-comment) ⋅ [docs](https://docs.trunk.io/ci-analytics/pull-request-metrics?utm_source=gh-comment) ⋅ learn more about [trunk.io](https://trunk.io?utm_source=gh-comment)</sub>
author_association: NONE
user: login: georgemitenkov
created_at: 2025-03-10T11:58:43Z
body: * **#16090** <a href="https://app.graphite.dev/github/pr/aptos-labs/aptos-core/16090?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a> 👈 <a href="https://app.graphite.dev/github/pr/aptos-labs/aptos-core/16090?utm_source=stack-comment-view-in-graphite" target="_blank">(View in Graphite)</a>
* **#16088** <a href="https://app.graphite.dev/github/pr/aptos-labs/aptos-core/16088?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a>: 1 other dependent PR ([#16115](https://github.com/aptos-labs/aptos-core/pull/16115) <a href="https://app.graphite.dev/github/pr/aptos-labs/aptos-core/16115?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a>)
* **#16087** <a href="https://app.graphite.dev/github/pr/aptos-labs/aptos-core/16087?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a>
* `main`


<h2></h2>

This stack of pull requests is managed by <a href="https://graphite.dev?utm-source=stack-comment"><b>Graphite</b></a>. Learn more about <a href="https://stacking.dev/?utm_source=stack-comment">stacking</a>.
<!-- Current dependencies on/for this PR: -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-17T19:25:17Z
body: ### Forge is running suite `framework_upgrade` on `82240c9c7137087c575bf5d670abfa0dddc3ae9f` ==> `e01e5af8c17d83f607e356a9eb0f8725a2716327`
* [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-framework-upgrade-pr-16090&var-metrics_source=All&var-chain_name=forge-0&refresh=10s&from=now-15m&to=now)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-framework-upgrade-pr-16090%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-framework-upgrade-pr-16090%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16090%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16090%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22quickRange%22%3A%20%2230m%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16090&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16090&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/13907822507)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-framework-upgrade -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-17T19:25:19Z
body: ### Forge is running suite `realistic_env_max_load` on `e01e5af8c17d83f607e356a9eb0f8725a2716327`
* [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-e2e-pr-16090&var-metrics_source=All&var-chain_name=forge-1&refresh=10s&from=now-15m&to=now)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-e2e-pr-16090%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-e2e-pr-16090%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16090%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16090%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22quickRange%22%3A%20%2230m%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16090&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16090&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/13907822507)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-e2e -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-17T19:25:22Z
body: ### Forge is running suite `compat` on `82240c9c7137087c575bf5d670abfa0dddc3ae9f` ==> `e01e5af8c17d83f607e356a9eb0f8725a2716327`
* [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-compat-pr-16090&var-metrics_source=All&var-chain_name=forge-0&refresh=10s&from=now-15m&to=now)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-compat-pr-16090%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-compat-pr-16090%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16090%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16090%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22quickRange%22%3A%20%2230m%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16090&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16090&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/13907822507)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-compat -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-17T19:37:25Z
body: ### :white_check_mark: Forge suite `compat` success on `82240c9c7137087c575bf5d670abfa0dddc3ae9f` ==> `e01e5af8c17d83f607e356a9eb0f8725a2716327`
```
Compatibility test results for 82240c9c7137087c575bf5d670abfa0dddc3ae9f ==> e01e5af8c17d83f607e356a9eb0f8725a2716327 (PR)
1. Check liveness of validators at old version: 82240c9c7137087c575bf5d670abfa0dddc3ae9f
compatibility::simple-validator-upgrade::liveness-check : committed: 5611.94 txn/s, submitted: 5612.11 txn/s, expired: 0.17 txn/s, latency: 2925.48 ms, (p50: 2900 ms, p70: 3200, p90: 3700 ms, p99: 4200 ms), latency samples: 356009
2. Upgrading first Validator to new version: e01e5af8c17d83f607e356a9eb0f8725a2716327
compatibility::simple-validator-upgrade::single-validator-upgrading : committed: 2672.66 txn/s, latency: 10918.39 ms, (p50: 11900 ms, p70: 13500, p90: 14000 ms, p99: 14100 ms), latency samples: 59880
compatibility::simple-validator-upgrade::single-validator-upgrade : committed: 2676.89 txn/s, latency: 12459.21 ms, (p50: 14000 ms, p70: 14500, p90: 14600 ms, p99: 14800 ms), latency samples: 99960
3. Upgrading rest of first batch to new version: e01e5af8c17d83f607e356a9eb0f8725a2716327
compatibility::simple-validator-upgrade::half-validator-upgrading : committed: 2954.59 txn/s, latency: 9850.65 ms, (p50: 10500 ms, p70: 12100, p90: 12700 ms, p99: 12800 ms), latency samples: 65520
compatibility::simple-validator-upgrade::half-validator-upgrade : committed: 2928.46 txn/s, latency: 11426.49 ms, (p50: 12600 ms, p70: 13200, p90: 13500 ms, p99: 13500 ms), latency samples: 111640
4. upgrading second batch to new version: e01e5af8c17d83f607e356a9eb0f8725a2716327
compatibility::simple-validator-upgrade::rest-validator-upgrading : committed: 5263.84 txn/s, latency: 5757.50 ms, (p50: 6500 ms, p70: 6700, p90: 7800 ms, p99: 8000 ms), latency samples: 105320
compatibility::simple-validator-upgrade::rest-validator-upgrade : committed: 5365.12 txn/s, latency: 6385.21 ms, (p50: 6900 ms, p70: 7200, p90: 7600 ms, p99: 7700 ms), latency samples: 185000
5. check swarm health
Compatibility test for 82240c9c7137087c575bf5d670abfa0dddc3ae9f ==> e01e5af8c17d83f607e356a9eb0f8725a2716327 passed
Test Ok
```
* [Grafana dashboard](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-compat-pr-16090&var-metrics_source=All&var-chain_name=forge-0&from=1742239529000&to=1742240244000)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-compat-pr-16090%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-compat-pr-16090%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1742239529000&end=1742240244000)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16090%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16090%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22startTime%22%3A%20%222025-03-17T19%3A25%3A29.203309%2B00%3A00%22%2C%20%22endTime%22%3A%20%222025-03-17T19%3A37%3A24.232327%2B00%3A00%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1742239529&until=1742240244&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16090&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1742239529&until=1742240244&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16090&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/13907822507)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-compat -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-17T19:37:48Z
body: ### :white_check_mark: Forge suite `framework_upgrade` success on `82240c9c7137087c575bf5d670abfa0dddc3ae9f` ==> `e01e5af8c17d83f607e356a9eb0f8725a2716327`
```
Compatibility test results for 82240c9c7137087c575bf5d670abfa0dddc3ae9f ==> e01e5af8c17d83f607e356a9eb0f8725a2716327 (PR)
Upgrade the nodes to version: e01e5af8c17d83f607e356a9eb0f8725a2716327
framework_upgrade::framework-upgrade::full-framework-upgrade : committed: 1751.03 txn/s, submitted: 1756.35 txn/s, failed submission: 5.32 txn/s, expired: 5.32 txn/s, latency: 1696.40 ms, (p50: 1500 ms, p70: 1800, p90: 2200 ms, p99: 3600 ms), latency samples: 157940
Upgrade the remaining nodes to version: e01e5af8c17d83f607e356a9eb0f8725a2716327
framework_upgrade::framework-upgrade::full-framework-upgrade : committed: 1734.61 txn/s, submitted: 1740.77 txn/s, failed submission: 6.15 txn/s, expired: 6.15 txn/s, latency: 1707.90 ms, (p50: 1700 ms, p70: 2000, p90: 2400 ms, p99: 3300 ms), latency samples: 157822
5. check swarm health
Compatibility test for 82240c9c7137087c575bf5d670abfa0dddc3ae9f ==> e01e5af8c17d83f607e356a9eb0f8725a2716327 passed
framework_upgrade::framework-upgrade::full-framework-upgrade : committed: 1810.95 txn/s, submitted: 1817.34 txn/s, failed submission: 6.40 txn/s, expired: 6.40 txn/s, latency: 1634.81 ms, (p50: 1500 ms, p70: 1800, p90: 2100 ms, p99: 3000 ms), latency samples: 164240
Test Ok
```
* [Grafana dashboard](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-framework-upgrade-pr-16090&var-metrics_source=All&var-chain_name=forge-0&from=1742239522000&to=1742240266000)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-framework-upgrade-pr-16090%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-framework-upgrade-pr-16090%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1742239522000&end=1742240266000)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16090%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16090%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22startTime%22%3A%20%222025-03-17T19%3A25%3A22.698807%2B00%3A00%22%2C%20%22endTime%22%3A%20%222025-03-17T19%3A37%3A46.847844%2B00%3A00%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1742239522&until=1742240266&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16090&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1742239522&until=1742240266&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16090&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/13907822507)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-framework-upgrade -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-17T19:38:06Z
body: ### :white_check_mark: Forge suite `realistic_env_max_load` success on `e01e5af8c17d83f607e356a9eb0f8725a2716327`
```
two traffics test: inner traffic : committed: 11001.46 txn/s, submitted: 11048.33 txn/s, expired: 46.87 txn/s, latency: 3363.23 ms, (p50: 3300 ms, p70: 3600, p90: 3900 ms, p99: 7500 ms), latency samples: 4183000
two traffics test : committed: 100.00 txn/s, latency: 1659.47 ms, (p50: 1600 ms, p70: 1800, p90: 2000 ms, p99: 3000 ms), latency samples: 1940
Latency breakdown for phase 0: ["MempoolToBlockCreation: max: 2.255, avg: 2.027", "ConsensusProposalToOrdered: max: 0.323, avg: 0.316", "ConsensusOrderedToCommit: max: 0.748, avg: 0.541", "ConsensusProposalToCommit: max: 1.067, avg: 0.857"]
Max non-epoch-change gap was: 0 rounds at version 0 (avg 0.00) [limit 4], 1.18s no progress at version 1900010 (avg 0.22s) [limit 15].
Max epoch-change gap was: 0 rounds at version 0 (avg 0.00) [limit 4], 1.27s no progress at version 1895798 (avg 0.99s) [limit 16].
Test Ok
```
* [Grafana dashboard](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-e2e-pr-16090&var-metrics_source=All&var-chain_name=forge-1&from=1742239525000&to=1742240284000)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-e2e-pr-16090%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-e2e-pr-16090%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1742239525000&end=1742240284000)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16090%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16090%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22startTime%22%3A%20%222025-03-17T19%3A25%3A25.712407%2B00%3A00%22%2C%20%22endTime%22%3A%20%222025-03-17T19%3A38%3A04.531771%2B00%3A00%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1742239525&until=1742240284&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16090&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1742239525&until=1742240284&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16090&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/13907822507)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-e2e -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-18T12:05:07Z
body: ### Forge is running suite `compat` on `82240c9c7137087c575bf5d670abfa0dddc3ae9f` ==> `4063166850f557b000486f280c6b072f60b7c8db`
* [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-compat-pr-16090&var-metrics_source=All&var-chain_name=forge-0&refresh=10s&from=now-15m&to=now)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-compat-pr-16090%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-compat-pr-16090%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16090%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16090%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22quickRange%22%3A%20%2230m%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16090&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16090&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/13922480657)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-compat -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-18T12:05:09Z
body: ### Forge is running suite `framework_upgrade` on `82240c9c7137087c575bf5d670abfa0dddc3ae9f` ==> `4063166850f557b000486f280c6b072f60b7c8db`
* [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-framework-upgrade-pr-16090&var-metrics_source=All&var-chain_name=forge-0&refresh=10s&from=now-15m&to=now)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-framework-upgrade-pr-16090%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-framework-upgrade-pr-16090%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16090%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16090%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22quickRange%22%3A%20%2230m%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16090&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16090&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/13922480657)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-framework-upgrade -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-18T12:05:12Z
body: ### Forge is running suite `realistic_env_max_load` on `4063166850f557b000486f280c6b072f60b7c8db`
* [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-e2e-pr-16090&var-metrics_source=All&var-chain_name=forge-1&refresh=10s&from=now-15m&to=now)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-e2e-pr-16090%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-e2e-pr-16090%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16090%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16090%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22quickRange%22%3A%20%2230m%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16090&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16090&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/13922480657)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-e2e -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-18T12:16:22Z
body: ### :white_check_mark: Forge suite `realistic_env_max_load` success on `4063166850f557b000486f280c6b072f60b7c8db`
```
two traffics test: inner traffic : committed: 12760.45 txn/s, latency: 3118.14 ms, (p50: 2800 ms, p70: 3300, p90: 4200 ms, p99: 6600 ms), latency samples: 4851800
two traffics test : committed: 99.98 txn/s, latency: 3081.35 ms, (p50: 2800 ms, p70: 3200, p90: 5000 ms, p99: 6800 ms), latency samples: 1720
Latency breakdown for phase 0: ["MempoolToBlockCreation: max: 1.670, avg: 0.636", "ConsensusProposalToOrdered: max: 0.305, avg: 0.304", "ConsensusOrderedToCommit: max: 0.600, avg: 0.467", "ConsensusProposalToCommit: max: 0.903, avg: 0.770"]
Max non-epoch-change gap was: 1 rounds at version 34097 (avg 0.00) [limit 4], 2.07s no progress at version 34097 (avg 0.21s) [limit 15].
Max epoch-change gap was: 0 rounds at version 0 (avg 0.00) [limit 4], 0.98s no progress at version 2607198 (avg 0.98s) [limit 16].
Test Ok
```
* [Grafana dashboard](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-e2e-pr-16090&var-metrics_source=All&var-chain_name=forge-1&from=1742299518000&to=1742300180000)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-e2e-pr-16090%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-e2e-pr-16090%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1742299518000&end=1742300180000)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16090%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16090%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22startTime%22%3A%20%222025-03-18T12%3A05%3A18.544540%2B00%3A00%22%2C%20%22endTime%22%3A%20%222025-03-18T12%3A16%3A20.562369%2B00%3A00%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1742299518&until=1742300180&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16090&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1742299518&until=1742300180&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16090&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/13922480657)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-e2e -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-18T12:19:50Z
body: ### :white_check_mark: Forge suite `framework_upgrade` success on `82240c9c7137087c575bf5d670abfa0dddc3ae9f` ==> `4063166850f557b000486f280c6b072f60b7c8db`
```
Compatibility test results for 82240c9c7137087c575bf5d670abfa0dddc3ae9f ==> 4063166850f557b000486f280c6b072f60b7c8db (PR)
Upgrade the nodes to version: 4063166850f557b000486f280c6b072f60b7c8db
framework_upgrade::framework-upgrade::full-framework-upgrade : committed: 1656.59 txn/s, submitted: 1661.88 txn/s, failed submission: 5.29 txn/s, expired: 5.29 txn/s, latency: 1766.15 ms, (p50: 1600 ms, p70: 1900, p90: 2600 ms, p99: 4300 ms), latency samples: 150280
Upgrade the remaining nodes to version: 4063166850f557b000486f280c6b072f60b7c8db
framework_upgrade::framework-upgrade::full-framework-upgrade : committed: 1654.12 txn/s, submitted: 1658.75 txn/s, failed submission: 4.62 txn/s, expired: 4.62 txn/s, latency: 1783.47 ms, (p50: 1700 ms, p70: 1900, p90: 2400 ms, p99: 3600 ms), latency samples: 150300
5. check swarm health
Compatibility test for 82240c9c7137087c575bf5d670abfa0dddc3ae9f ==> 4063166850f557b000486f280c6b072f60b7c8db passed
framework_upgrade::framework-upgrade::full-framework-upgrade : committed: 1599.36 txn/s, submitted: 1603.27 txn/s, failed submission: 3.92 txn/s, expired: 3.92 txn/s, latency: 2403.32 ms, (p50: 1700 ms, p70: 2000, p90: 2700 ms, p99: 12400 ms), latency samples: 138841
Test Ok
```
* [Grafana dashboard](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-framework-upgrade-pr-16090&var-metrics_source=All&var-chain_name=forge-0&from=1742299515000&to=1742300388000)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-framework-upgrade-pr-16090%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-framework-upgrade-pr-16090%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1742299515000&end=1742300388000)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16090%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16090%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22startTime%22%3A%20%222025-03-18T12%3A05%3A15.943797%2B00%3A00%22%2C%20%22endTime%22%3A%20%222025-03-18T12%3A19%3A48.516029%2B00%3A00%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1742299515&until=1742300388&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16090&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1742299515&until=1742300388&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16090&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/13922480657)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-framework-upgrade -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-18T12:19:50Z
body: ### :white_check_mark: Forge suite `compat` success on `82240c9c7137087c575bf5d670abfa0dddc3ae9f` ==> `4063166850f557b000486f280c6b072f60b7c8db`
```
Compatibility test results for 82240c9c7137087c575bf5d670abfa0dddc3ae9f ==> 4063166850f557b000486f280c6b072f60b7c8db (PR)
1. Check liveness of validators at old version: 82240c9c7137087c575bf5d670abfa0dddc3ae9f
compatibility::simple-validator-upgrade::liveness-check : committed: 5695.04 txn/s, submitted: 5695.21 txn/s, expired: 0.17 txn/s, latency: 2793.24 ms, (p50: 2700 ms, p70: 3000, p90: 3500 ms, p99: 4400 ms), latency samples: 371889
2. Upgrading first Validator to new version: 4063166850f557b000486f280c6b072f60b7c8db
compatibility::simple-validator-upgrade::single-validator-upgrading : committed: 2989.23 txn/s, latency: 9772.39 ms, (p50: 10400 ms, p70: 11900, p90: 12600 ms, p99: 12800 ms), latency samples: 66020
compatibility::simple-validator-upgrade::single-validator-upgrade : committed: 2970.58 txn/s, latency: 11238.24 ms, (p50: 12400 ms, p70: 13000, p90: 13300 ms, p99: 13300 ms), latency samples: 113160
3. Upgrading rest of first batch to new version: 4063166850f557b000486f280c6b072f60b7c8db
compatibility::simple-validator-upgrade::half-validator-upgrading : committed: 2880.31 txn/s, latency: 10092.23 ms, (p50: 11000 ms, p70: 12500, p90: 12800 ms, p99: 13000 ms), latency samples: 62440
compatibility::simple-validator-upgrade::half-validator-upgrade : committed: 2913.26 txn/s, latency: 11409.02 ms, (p50: 12800 ms, p70: 13100, p90: 13200 ms, p99: 13500 ms), latency samples: 111740
4. upgrading second batch to new version: 4063166850f557b000486f280c6b072f60b7c8db
compatibility::simple-validator-upgrade::rest-validator-upgrading : committed: 5201.91 txn/s, latency: 5780.94 ms, (p50: 6600 ms, p70: 6800, p90: 7700 ms, p99: 7800 ms), latency samples: 103860
compatibility::simple-validator-upgrade::rest-validator-upgrade : committed: 5106.75 txn/s, latency: 6597.05 ms, (p50: 6800 ms, p70: 7600, p90: 7900 ms, p99: 8200 ms), latency samples: 182060
5. check swarm health
Compatibility test for 82240c9c7137087c575bf5d670abfa0dddc3ae9f ==> 4063166850f557b000486f280c6b072f60b7c8db passed
Test Ok
```
* [Grafana dashboard](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-compat-pr-16090&var-metrics_source=All&var-chain_name=forge-0&from=1742299514000&to=1742300389000)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-compat-pr-16090%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-compat-pr-16090%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1742299514000&end=1742300389000)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16090%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16090%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22startTime%22%3A%20%222025-03-18T12%3A05%3A14.392679%2B00%3A00%22%2C%20%22endTime%22%3A%20%222025-03-18T12%3A19%3A49.191149%2B00%3A00%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1742299514&until=1742300389&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16090&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1742299514&until=1742300389&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16090&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/13922480657)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-compat -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-18T20:17:12Z
body: ### Forge is running suite `realistic_env_max_load` on `63c209bc68a9ccded82a3aac1321e418488f8292`
* [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-e2e-pr-16090&var-metrics_source=All&var-chain_name=forge-1&refresh=10s&from=now-15m&to=now)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-e2e-pr-16090%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-e2e-pr-16090%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16090%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16090%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22quickRange%22%3A%20%2230m%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16090&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16090&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/13932378077)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-e2e -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-18T20:17:16Z
body: ### Forge is running suite `framework_upgrade` on `82240c9c7137087c575bf5d670abfa0dddc3ae9f` ==> `63c209bc68a9ccded82a3aac1321e418488f8292`
* [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-framework-upgrade-pr-16090&var-metrics_source=All&var-chain_name=forge-0&refresh=10s&from=now-15m&to=now)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-framework-upgrade-pr-16090%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-framework-upgrade-pr-16090%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16090%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16090%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22quickRange%22%3A%20%2230m%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16090&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16090&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/13932378077)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-framework-upgrade -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-18T20:17:17Z
body: ### Forge is running suite `compat` on `82240c9c7137087c575bf5d670abfa0dddc3ae9f` ==> `63c209bc68a9ccded82a3aac1321e418488f8292`
* [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-compat-pr-16090&var-metrics_source=All&var-chain_name=forge-0&refresh=10s&from=now-15m&to=now)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-compat-pr-16090%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-compat-pr-16090%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16090%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16090%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22quickRange%22%3A%20%2230m%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16090&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16090&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/13932378077)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-compat -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-18T20:29:19Z
body: ### :white_check_mark: Forge suite `compat` success on `82240c9c7137087c575bf5d670abfa0dddc3ae9f` ==> `63c209bc68a9ccded82a3aac1321e418488f8292`
```
Compatibility test results for 82240c9c7137087c575bf5d670abfa0dddc3ae9f ==> 63c209bc68a9ccded82a3aac1321e418488f8292 (PR)
1. Check liveness of validators at old version: 82240c9c7137087c575bf5d670abfa0dddc3ae9f
compatibility::simple-validator-upgrade::liveness-check : committed: 3699.78 txn/s, submitted: 3700.05 txn/s, expired: 0.27 txn/s, latency: 3676.61 ms, (p50: 3800 ms, p70: 4100, p90: 4600 ms, p99: 5000 ms), latency samples: 300338
2. Upgrading first Validator to new version: 63c209bc68a9ccded82a3aac1321e418488f8292
compatibility::simple-validator-upgrade::single-validator-upgrading : committed: 2472.78 txn/s, latency: 11871.53 ms, (p50: 12800 ms, p70: 14600, p90: 14900 ms, p99: 15100 ms), latency samples: 57000
compatibility::simple-validator-upgrade::single-validator-upgrade : committed: 2380.53 txn/s, latency: 14213.22 ms, (p50: 15400 ms, p70: 16200, p90: 16500 ms, p99: 16700 ms), latency samples: 90400
3. Upgrading rest of first batch to new version: 63c209bc68a9ccded82a3aac1321e418488f8292
compatibility::simple-validator-upgrade::half-validator-upgrading : committed: 2940.75 txn/s, latency: 9263.37 ms, (p50: 11200 ms, p70: 11500, p90: 11800 ms, p99: 11800 ms), latency samples: 61880
compatibility::simple-validator-upgrade::half-validator-upgrade : committed: 2857.59 txn/s, latency: 11674.79 ms, (p50: 13200 ms, p70: 13600, p90: 13700 ms, p99: 13800 ms), latency samples: 106980
4. upgrading second batch to new version: 63c209bc68a9ccded82a3aac1321e418488f8292
compatibility::simple-validator-upgrade::rest-validator-upgrading : committed: 4888.82 txn/s, latency: 6249.67 ms, (p50: 7100 ms, p70: 7500, p90: 7900 ms, p99: 8000 ms), latency samples: 98620
compatibility::simple-validator-upgrade::rest-validator-upgrade : committed: 4877.79 txn/s, latency: 7013.83 ms, (p50: 7600 ms, p70: 7900, p90: 8000 ms, p99: 8300 ms), latency samples: 174900
5. check swarm health
Compatibility test for 82240c9c7137087c575bf5d670abfa0dddc3ae9f ==> 63c209bc68a9ccded82a3aac1321e418488f8292 passed
Test Ok
```
* [Grafana dashboard](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-compat-pr-16090&var-metrics_source=All&var-chain_name=forge-0&from=1742329044000&to=1742329757000)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-compat-pr-16090%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-compat-pr-16090%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1742329044000&end=1742329757000)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16090%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16090%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22startTime%22%3A%20%222025-03-18T20%3A17%3A24.218762%2B00%3A00%22%2C%20%22endTime%22%3A%20%222025-03-18T20%3A29%3A17.549099%2B00%3A00%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1742329044&until=1742329757&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16090&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1742329044&until=1742329757&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16090&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/13932378077)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-compat -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-18T20:29:22Z
body: ### :white_check_mark: Forge suite `realistic_env_max_load` success on `63c209bc68a9ccded82a3aac1321e418488f8292`
```
two traffics test: inner traffic : committed: 11624.15 txn/s, submitted: 11630.67 txn/s, expired: 6.52 txn/s, latency: 3386.50 ms, (p50: 3000 ms, p70: 3200, p90: 4300 ms, p99: 10900 ms), latency samples: 4419820
two traffics test : committed: 100.00 txn/s, latency: 2377.92 ms, (p50: 1600 ms, p70: 2600, p90: 3700 ms, p99: 11700 ms), latency samples: 1820
Latency breakdown for phase 0: ["MempoolToBlockCreation: max: 2.295, avg: 1.378", "ConsensusProposalToOrdered: max: 0.325, avg: 0.320", "ConsensusOrderedToCommit: max: 0.437, avg: 0.411", "ConsensusProposalToCommit: max: 0.753, avg: 0.731"]
Max non-epoch-change gap was: 1 rounds at version 2077212 (avg 0.00) [limit 4], 2.27s no progress at version 2077212 (avg 0.22s) [limit 15].
Max epoch-change gap was: 0 rounds at version 0 (avg 0.00) [limit 4], 0.96s no progress at version 2077190 (avg 0.96s) [limit 16].
Test Ok
```
* [Grafana dashboard](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-e2e-pr-16090&var-metrics_source=All&var-chain_name=forge-1&from=1742329038000&to=1742329761000)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-e2e-pr-16090%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-e2e-pr-16090%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1742329038000&end=1742329761000)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16090%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16090%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22startTime%22%3A%20%222025-03-18T20%3A17%3A18.624949%2B00%3A00%22%2C%20%22endTime%22%3A%20%222025-03-18T20%3A29%3A21.108317%2B00%3A00%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1742329038&until=1742329761&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16090&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1742329038&until=1742329761&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16090&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/13932378077)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-e2e -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-18T20:31:02Z
body: ### :white_check_mark: Forge suite `framework_upgrade` success on `82240c9c7137087c575bf5d670abfa0dddc3ae9f` ==> `63c209bc68a9ccded82a3aac1321e418488f8292`
```
Compatibility test results for 82240c9c7137087c575bf5d670abfa0dddc3ae9f ==> 63c209bc68a9ccded82a3aac1321e418488f8292 (PR)
Upgrade the nodes to version: 63c209bc68a9ccded82a3aac1321e418488f8292
framework_upgrade::framework-upgrade::full-framework-upgrade : committed: 1403.85 txn/s, submitted: 1409.36 txn/s, failed submission: 5.51 txn/s, expired: 5.51 txn/s, latency: 2094.59 ms, (p50: 1800 ms, p70: 2100, p90: 3300 ms, p99: 6500 ms), latency samples: 127360
Upgrade the remaining nodes to version: 63c209bc68a9ccded82a3aac1321e418488f8292
framework_upgrade::framework-upgrade::full-framework-upgrade : committed: 1801.43 txn/s, submitted: 1806.74 txn/s, failed submission: 5.31 txn/s, expired: 5.31 txn/s, latency: 1636.48 ms, (p50: 1500 ms, p70: 1800, p90: 2500 ms, p99: 3300 ms), latency samples: 162740
5. check swarm health
Compatibility test for 82240c9c7137087c575bf5d670abfa0dddc3ae9f ==> 63c209bc68a9ccded82a3aac1321e418488f8292 passed
framework_upgrade::framework-upgrade::full-framework-upgrade : committed: 1621.36 txn/s, submitted: 1626.89 txn/s, failed submission: 5.53 txn/s, expired: 5.53 txn/s, latency: 1802.04 ms, (p50: 1600 ms, p70: 1900, p90: 2500 ms, p99: 3300 ms), latency samples: 146681
Test Ok
```
* [Grafana dashboard](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-framework-upgrade-pr-16090&var-metrics_source=All&var-chain_name=forge-0&from=1742329041000&to=1742329860000)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-framework-upgrade-pr-16090%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-framework-upgrade-pr-16090%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1742329041000&end=1742329860000)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16090%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16090%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22startTime%22%3A%20%222025-03-18T20%3A17%3A21.892415%2B00%3A00%22%2C%20%22endTime%22%3A%20%222025-03-18T20%3A31%3A00.795588%2B00%3A00%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1742329041&until=1742329860&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16090&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1742329041&until=1742329860&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16090&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/13932378077)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-framework-upgrade -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-26T08:30:57Z
body: ### Forge is running suite `realistic_env_max_load` on `ae6143b576b242496b86a7e35bf32c16cac85e12`
* [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-e2e-pr-16090&var-metrics_source=All&var-chain_name=forge-1&refresh=10s&from=now-15m&to=now)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-e2e-pr-16090%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-e2e-pr-16090%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16090%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16090%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22quickRange%22%3A%20%2230m%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16090&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16090&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14078468580)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-e2e -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-26T08:31:02Z
body: ### Forge is running suite `framework_upgrade` on `2e48f76143b9b0624af00b952048444fb9af0ca3` ==> `ae6143b576b242496b86a7e35bf32c16cac85e12`
* [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-framework-upgrade-pr-16090&var-metrics_source=All&var-chain_name=forge-0&refresh=10s&from=now-15m&to=now)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-framework-upgrade-pr-16090%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-framework-upgrade-pr-16090%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16090%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16090%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22quickRange%22%3A%20%2230m%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16090&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16090&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14078468580)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-framework-upgrade -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-26T08:31:03Z
body: ### Forge is running suite `compat` on `2e48f76143b9b0624af00b952048444fb9af0ca3` ==> `ae6143b576b242496b86a7e35bf32c16cac85e12`
* [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-compat-pr-16090&var-metrics_source=All&var-chain_name=forge-0&refresh=10s&from=now-15m&to=now)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-compat-pr-16090%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-compat-pr-16090%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16090%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16090%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22quickRange%22%3A%20%2230m%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16090&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16090&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14078468580)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-compat -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-26T08:43:21Z
body: ### :white_check_mark: Forge suite `realistic_env_max_load` success on `ae6143b576b242496b86a7e35bf32c16cac85e12`
```
two traffics test: inner traffic : committed: 10971.45 txn/s, latency: 3620.45 ms, (p50: 3600 ms, p70: 3600, p90: 4100 ms, p99: 5100 ms), latency samples: 4171580
two traffics test : committed: 99.96 txn/s, latency: 1605.57 ms, (p50: 1400 ms, p70: 1600, p90: 2200 ms, p99: 2800 ms), latency samples: 1860
Latency breakdown for phase 0: ["MempoolToBlockCreation: max: 2.755, avg: 2.283", "ConsensusProposalToOrdered: max: 0.347, avg: 0.332", "ConsensusOrderedToCommit: max: 0.415, avg: 0.394", "ConsensusProposalToCommit: max: 0.755, avg: 0.726"]
Max non-epoch-change gap was: 0 rounds at version 0 (avg 0.00) [limit 4], 1.27s no progress at version 1879517 (avg 0.23s) [limit 15].
Max epoch-change gap was: 0 rounds at version 0 (avg 0.00) [limit 4], 0.86s no progress at version 1864605 (avg 0.86s) [limit 16].
Test Ok
```
* [Grafana dashboard](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-e2e-pr-16090&var-metrics_source=All&var-chain_name=forge-1&from=1742977864000&to=1742978599000)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-e2e-pr-16090%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-e2e-pr-16090%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1742977864000&end=1742978599000)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16090%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16090%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22startTime%22%3A%20%222025-03-26T08%3A31%3A04.348169%2B00%3A00%22%2C%20%22endTime%22%3A%20%222025-03-26T08%3A43%3A19.066196%2B00%3A00%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1742977864&until=1742978599&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16090&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1742977864&until=1742978599&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16090&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14078468580)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-e2e -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-26T08:43:36Z
body: ### :white_check_mark: Forge suite `compat` success on `2e48f76143b9b0624af00b952048444fb9af0ca3` ==> `ae6143b576b242496b86a7e35bf32c16cac85e12`
```
Compatibility test results for 2e48f76143b9b0624af00b952048444fb9af0ca3 ==> ae6143b576b242496b86a7e35bf32c16cac85e12 (PR)
1. Check liveness of validators at old version: 2e48f76143b9b0624af00b952048444fb9af0ca3
compatibility::simple-validator-upgrade::liveness-check : committed: 5588.75 txn/s, submitted: 5589.09 txn/s, expired: 0.34 txn/s, latency: 2891.97 ms, (p50: 2900 ms, p70: 3100, p90: 3500 ms, p99: 4300 ms), latency samples: 362978
2. Upgrading first Validator to new version: ae6143b576b242496b86a7e35bf32c16cac85e12
compatibility::simple-validator-upgrade::single-validator-upgrading : committed: 3049.75 txn/s, latency: 10507.16 ms, (p50: 9900 ms, p70: 12700, p90: 13000 ms, p99: 13000 ms), latency samples: 84000
compatibility::simple-validator-upgrade::single-validator-upgrade : committed: 2767.10 txn/s, latency: 11919.27 ms, (p50: 13300 ms, p70: 13600, p90: 13900 ms, p99: 14000 ms), latency samples: 106600
3. Upgrading rest of first batch to new version: ae6143b576b242496b86a7e35bf32c16cac85e12
compatibility::simple-validator-upgrade::half-validator-upgrading : committed: 2880.00 txn/s, latency: 10146.45 ms, (p50: 10900 ms, p70: 12600, p90: 13200 ms, p99: 13400 ms), latency samples: 63780
compatibility::simple-validator-upgrade::half-validator-upgrade : committed: 2912.17 txn/s, latency: 11438.01 ms, (p50: 12900 ms, p70: 13200, p90: 13400 ms, p99: 13700 ms), latency samples: 110560
4. upgrading second batch to new version: ae6143b576b242496b86a7e35bf32c16cac85e12
compatibility::simple-validator-upgrade::rest-validator-upgrading : committed: 5714.79 txn/s, latency: 5374.64 ms, (p50: 6200 ms, p70: 6600, p90: 6700 ms, p99: 6800 ms), latency samples: 108640
compatibility::simple-validator-upgrade::rest-validator-upgrade : committed: 5609.19 txn/s, latency: 6115.12 ms, (p50: 6800 ms, p70: 6900, p90: 7100 ms, p99: 7200 ms), latency samples: 191220
5. check swarm health
Compatibility test for 2e48f76143b9b0624af00b952048444fb9af0ca3 ==> ae6143b576b242496b86a7e35bf32c16cac85e12 passed
Test Ok
```
* [Grafana dashboard](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-compat-pr-16090&var-metrics_source=All&var-chain_name=forge-0&from=1742977870000&to=1742978614000)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-compat-pr-16090%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-compat-pr-16090%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1742977870000&end=1742978614000)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16090%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16090%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22startTime%22%3A%20%222025-03-26T08%3A31%3A10.687440%2B00%3A00%22%2C%20%22endTime%22%3A%20%222025-03-26T08%3A43%3A34.294937%2B00%3A00%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1742977870&until=1742978614&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16090&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1742977870&until=1742978614&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16090&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14078468580)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-compat -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-26T08:44:30Z
body: ### :white_check_mark: Forge suite `framework_upgrade` success on `2e48f76143b9b0624af00b952048444fb9af0ca3` ==> `ae6143b576b242496b86a7e35bf32c16cac85e12`
```
Compatibility test results for 2e48f76143b9b0624af00b952048444fb9af0ca3 ==> ae6143b576b242496b86a7e35bf32c16cac85e12 (PR)
Upgrade the nodes to version: ae6143b576b242496b86a7e35bf32c16cac85e12
framework_upgrade::framework-upgrade::full-framework-upgrade : committed: 1846.00 txn/s, submitted: 1852.28 txn/s, failed submission: 6.27 txn/s, expired: 6.27 txn/s, latency: 1606.93 ms, (p50: 1500 ms, p70: 1800, p90: 2100 ms, p99: 2500 ms), latency samples: 164761
Upgrade the remaining nodes to version: ae6143b576b242496b86a7e35bf32c16cac85e12
framework_upgrade::framework-upgrade::full-framework-upgrade : committed: 1723.27 txn/s, submitted: 1727.54 txn/s, failed submission: 4.27 txn/s, expired: 4.27 txn/s, latency: 1693.25 ms, (p50: 1500 ms, p70: 1800, p90: 2400 ms, p99: 3000 ms), latency samples: 153320
5. check swarm health
Compatibility test for 2e48f76143b9b0624af00b952048444fb9af0ca3 ==> ae6143b576b242496b86a7e35bf32c16cac85e12 passed
framework_upgrade::framework-upgrade::full-framework-upgrade : committed: 1653.09 txn/s, submitted: 1658.55 txn/s, failed submission: 5.46 txn/s, expired: 5.46 txn/s, latency: 2274.85 ms, (p50: 1500 ms, p70: 1800, p90: 2500 ms, p99: 13600 ms), latency samples: 139340
Test Ok
```
* [Grafana dashboard](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-framework-upgrade-pr-16090&var-metrics_source=All&var-chain_name=forge-0&from=1742977869000&to=1742978668000)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-framework-upgrade-pr-16090%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-framework-upgrade-pr-16090%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1742977869000&end=1742978668000)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16090%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16090%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22startTime%22%3A%20%222025-03-26T08%3A31%3A09.483059%2B00%3A00%22%2C%20%22endTime%22%3A%20%222025-03-26T08%3A44%3A28.712377%2B00%3A00%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1742977869&until=1742978668&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16090&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1742977869&until=1742978668&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16090&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14078468580)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-framework-upgrade -->
author_association: CONTRIBUTOR