Repository: aptos-labs/aptos-core
Pull request #16682: [vm] Refactor function type layout
Author: login: georgemitenkov
Created: 2025-05-23T09:03:42Z | Merged: 2025-05-30T09:38:54Z

## Description

Before, we defined function value layout as a triple of argument layouts, return value layouts and abilities. This design does not make sense because then it is not possible to store functions which operate on references (#16664):

```move
// Invariant violation returned at runtime, because reference layout is not possible.
struct Wrapper(|&signer|) has copy, drop, store, key;
```

In general, layout of function value is well defined as the function identifier, type arguments, and captured argument-layout pairs, so argument layouts and return value layouts are really not needed. In the end, here we talk about *layout*, not a *type*. Note that adding a "reference" variant to layout is not correct.

This PR changes function layout to a primitive `MoveTypeLayout::Function`. This works perfectly fine in the VM, as we do not care about argument or return types of a stored function (these are checked by paranoid mode at call site).

## How Has This Been Tested?

- Transactional test
- e2e Move test for aggregators

## Key Areas to Review
<!--
- Identify any critical parts of the code that require special attention or understanding. Explain why these parts are crucial to the functionality or architecture of the project.
- Point out any areas where complex logic has been implemented. Provide a brief explanation of the logic and your approach to make it easier for reviewers to follow.
- Highlight any areas where you are particularly concerned or unsure about the code's impact on the change. This can include potential performance or security issues, or compatibility with existing features.
-->

## Type of Change

- [x] New feature
- [x] Bug fix

## Which Components or Systems Does This Change Impact?

- [x] Move/Aptos Virtual Machine

## Checklist
- [x] I have read and followed the [CONTRIBUTING](https://github.com/aptos-labs/aptos-core/blob/main/CONTRIBUTING.md) doc
- [x] I have performed a self-review of my own code
- [x] I have commented my code, particularly in hard-to-understand areas
- [x] I identified and added all stakeholders and component owners affected by this change as reviewers
- [x] I tested both happy and unhappy path of the functionality
- [x] I have made corresponding changes to the documentation

<!-- Thank you for your contribution! -->



Diff:
diff --git a/api/types/src/convert.rs b/api/types/src/convert.rs
index 83b171a0290..7a73c28eece 100644
--- a/api/types/src/convert.rs
+++ b/api/types/src/convert.rs
@@ -927,7 +927,7 @@ impl<'a, S: StateView> MoveConverter<'a, S> {
             MoveTypeLayout::Struct(struct_layout) => {
                 self.try_into_vm_value_struct(struct_layout, val)?
             },
-            MoveTypeLayout::Function(..) => {
+            MoveTypeLayout::Function => {
                 // TODO(#15664): do we actually need this? It appears the code here is dead and
                 //   nowhere used
                 bail!("unexpected move type {:?} for value {:?}", layout, val)
diff --git a/aptos-move/e2e-move-tests/src/tests/aggregator_v2.data/function_values/sources/function_store.move b/aptos-move/e2e-move-tests/src/tests/aggregator_v2.data/function_values/sources/function_store.move
index 187c1e9ebbd..1811931d404 100644
--- a/aptos-move/e2e-move-tests/src/tests/aggregator_v2.data/function_values/sources/function_store.move
+++ b/aptos-move/e2e-move-tests/src/tests/aggregator_v2.data/function_values/sources/function_store.move
@@ -1,5 +1,5 @@
 module 0x1::function_store {
-    use aptos_framework::aggregator_v2::{Aggregator, create_unbounded_aggregator};
+    use aptos_framework::aggregator_v2::{Aggregator, create_unbounded_aggregator, try_add};
 
     struct FunctionStore has key, store {
         // Capturing aggregators, snapshots or anything that contains delayed fields is not
@@ -35,4 +35,27 @@ module 0x1::function_store {
     public fun view_function_store_exists(addr: address): bool {
         exists<FunctionStore>(addr)
     }
+
+    struct FunctionStoreV2 has key, store {
+        value: u64,
+        add: |&mut Aggregator<u64>|bool has copy + store,
+    }
+
+    public entry fun try_initialize_should_succeed(account: &signer, value: u64) {
+        let add = |a| try_add(a, value);
+        move_to(account, FunctionStoreV2 { value, add });
+    }
+
+    public entry fun run_stored_add(account: &signer, value: u64) acquires FunctionStoreV2 {
+        let aggregator = create_unbounded_aggregator<u64>();
+        aggregator.add(value);
+
+        let addr = std::signer::address_of(account);
+        let store = borrow_global<FunctionStoreV2>(addr);
+
+        let stored_add = store.add;
+        let success = stored_add(&mut aggregator);
+        assert!(success, 123);
+        assert!(aggregator.read() == value + store.value, 234);
+    }
 }
diff --git a/aptos-move/e2e-move-tests/src/tests/aggregator_v2_function_values.rs b/aptos-move/e2e-move-tests/src/tests/aggregator_v2_function_values.rs
index 4b8e81d1511..a9c52d8ba8d 100644
--- a/aptos-move/e2e-move-tests/src/tests/aggregator_v2_function_values.rs
+++ b/aptos-move/e2e-move-tests/src/tests/aggregator_v2_function_values.rs
@@ -112,6 +112,29 @@ fn test_function_value_captures_aggregator_is_not_storable() {
     assert!(!exists);
 }
 
+#[test]
+fn test_function_value_uses_aggregator_is_storable() {
+    let mut h = MoveHarness::new_with_executor(FakeExecutor::from_head_genesis().set_parallel());
+    let acc = h.new_account_at(AccountAddress::from_hex_literal("0x123").unwrap());
+    initialize(&mut h);
+
+    let status = h.run_entry_function(
+        &acc,
+        MemberId::from_str("0x1::function_store::try_initialize_should_succeed").unwrap(),
+        vec![],
+        vec![bcs::to_bytes(&100_u64).unwrap()],
+    );
+    assert_success!(status);
+
+    let status = h.run_entry_function(
+        &acc,
+        MemberId::from_str("0x1::function_store::run_stored_add").unwrap(),
+        vec![],
+        vec![bcs::to_bytes(&200_u64).unwrap()],
+    );
+    assert_success!(status);
+}
+
 #[test]
 fn test_function_value_captures_aggregator() {
     let mut h = MoveHarness::new_with_executor(FakeExecutor::from_head_genesis().set_parallel());
diff --git a/aptos-move/framework/move-stdlib/src/natives/bcs.rs b/aptos-move/framework/move-stdlib/src/natives/bcs.rs
index 88d5dc8d633..8781b367ae4 100644
--- a/aptos-move/framework/move-stdlib/src/natives/bcs.rs
+++ b/aptos-move/framework/move-stdlib/src/natives/bcs.rs
@@ -199,7 +199,7 @@ fn constant_serialized_size(ty_layout: &MoveTypeLayout) -> (u64, PartialVMResult
         MoveTypeLayout::Struct(
             MoveStructLayout::RuntimeVariants(_) | MoveStructLayout::WithVariants(_),
         )
-        | MoveTypeLayout::Function(..) => Ok(None),
+        | MoveTypeLayout::Function => Ok(None),
         MoveTypeLayout::Struct(MoveStructLayout::Runtime(fields)) => {
             let mut total = Some(0);
             for field in fields {
diff --git a/aptos-move/framework/src/natives/string_utils.rs b/aptos-move/framework/src/natives/string_utils.rs
index 97558bfa45d..8590d0b8f9f 100644
--- a/aptos-move/framework/src/natives/string_utils.rs
+++ b/aptos-move/framework/src/natives/string_utils.rs
@@ -358,7 +358,7 @@ fn native_format_impl(
             )?;
             out.push('}');
         },
-        MoveTypeLayout::Function(_) => {
+        MoveTypeLayout::Function => {
             // Notice that we print the undecorated value representation,
             // avoiding potential loading of the function to get full
             // decorated type information.
diff --git a/testsuite/fuzzer/fuzz/fuzz_targets/move/utils/helpers.rs b/testsuite/fuzzer/fuzz/fuzz_targets/move/utils/helpers.rs
index 52c3973dec3..bb0a4b5f31f 100644
--- a/testsuite/fuzzer/fuzz/fuzz_targets/move/utils/helpers.rs
+++ b/testsuite/fuzzer/fuzz/fuzz_targets/move/utils/helpers.rs
@@ -6,10 +6,7 @@
 use aptos_language_e2e_tests::{account::Account, executor::FakeExecutor};
 use arbitrary::Arbitrary;
 use move_binary_format::file_format::CompiledModule;
-use move_core_types::{
-    function::MoveFunctionLayout,
-    value::{MoveStructLayout, MoveTypeLayout},
-};
+use move_core_types::value::{MoveStructLayout, MoveTypeLayout};
 
 #[macro_export]
 macro_rules! tdbg {
@@ -71,9 +68,16 @@ pub(crate) fn is_valid_layout(layout: &MoveTypeLayout) -> bool {
     use MoveTypeLayout as L;
 
     match layout {
-        L::Bool | L::U8 | L::U16 | L::U32 | L::U64 | L::U128 | L::U256 | L::Address | L::Signer => {
-            true
-        },
+        L::Bool
+        | L::U8
+        | L::U16
+        | L::U32
+        | L::U64
+        | L::U128
+        | L::U256
+        | L::Address
+        | L::Signer
+        | L::Function => true,
 
         L::Vector(layout) | L::Native(_, layout) => is_valid_layout(layout),
         L::Struct(MoveStructLayout::RuntimeVariants(variants)) => {
@@ -85,9 +89,6 @@ pub(crate) fn is_valid_layout(layout: &MoveTypeLayout) -> bool {
             }
             fields.iter().all(is_valid_layout)
         },
-        L::Function(MoveFunctionLayout(args, results, _)) => {
-            args.iter().chain(results).all(is_valid_layout)
-        },
         L::Struct(_) => {
             // decorated layouts not supported
             false
diff --git a/third_party/move/move-binary-format/src/constant.rs b/third_party/move/move-binary-format/src/constant.rs
index bd6407714db..0c56bc1ce3f 100644
--- a/third_party/move/move-binary-format/src/constant.rs
+++ b/third_party/move/move-binary-format/src/constant.rs
@@ -40,7 +40,7 @@ fn construct_ty_for_constant(layout: &MoveTypeLayout) -> Option<SignatureToken>
             construct_ty_for_constant(l.as_ref())?,
         ))),
         MoveTypeLayout::Struct(_) => None,
-        MoveTypeLayout::Function(_) => None,
+        MoveTypeLayout::Function => None,
         MoveTypeLayout::Bool => Some(SignatureToken::Bool),
 
         // It is not possible to have native layout for constant values.
diff --git a/third_party/move/move-compiler-v2/transactional-tests/tests/no-v1-comparison/closures/storage_examples.exp b/third_party/move/move-compiler-v2/transactional-tests/tests/no-v1-comparison/closures/storage_examples.exp
new file mode 100644
index 00000000000..34aae29fea5
--- /dev/null
+++ b/third_party/move/move-compiler-v2/transactional-tests/tests/no-v1-comparison/closures/storage_examples.exp
@@ -0,0 +1,4 @@
+processed 3 tasks
+
+task 2 'run'. lines 23-23:
+return values: 7
diff --git a/third_party/move/move-compiler-v2/transactional-tests/tests/no-v1-comparison/closures/storage_examples.move b/third_party/move/move-compiler-v2/transactional-tests/tests/no-v1-comparison/closures/storage_examples.move
new file mode 100644
index 00000000000..64955a38710
--- /dev/null
+++ b/third_party/move/move-compiler-v2/transactional-tests/tests/no-v1-comparison/closures/storage_examples.move
@@ -0,0 +1,23 @@
+//# publish
+module 0xc0ffee::m {
+    struct Wrapper(|&signer|u64) has copy, drop, store, key;
+
+    #[persistent]
+    fun foo(_s: &signer): u64 {
+        7
+    }
+
+    public fun init(s: &signer) {
+        let w = Wrapper(foo);
+        move_to(s, w)
+    }
+
+    public fun fetch_and_run(s: &signer): u64{
+        let w = borrow_global<Wrapper>(std::signer::address_of(s));
+        (w.0)(s)
+    }
+}
+
+//# run 0xc0ffee::m::init --signers 0xc0ffee
+
+//# run 0xc0ffee::m::fetch_and_run --signers 0xc0ffee
diff --git a/third_party/move/move-core/types/src/function.rs b/third_party/move/move-core/types/src/function.rs
index 8a575cb5ced..80c4e1c9536 100644
--- a/third_party/move/move-core/types/src/function.rs
+++ b/third_party/move/move-core/types/src/function.rs
@@ -4,7 +4,7 @@
 use crate::{
     ability::AbilitySet,
     identifier::Identifier,
-    language_storage::{FunctionTag, ModuleId, TypeTag},
+    language_storage::{ModuleId, TypeTag},
     value::{MoveTypeLayout, MoveValue},
 };
 use serde::{de::Error, ser::SerializeSeq, Deserialize, Serialize};
@@ -253,10 +253,9 @@ pub struct MoveClosure {
     pub captured: Vec<(MoveTypeLayout, MoveValue)>,
 }
 
-#[allow(unused)] // Currently, we do not use the expected function layout
-pub(crate) struct ClosureVisitor<'a>(pub(crate) &'a MoveFunctionLayout);
+pub(crate) struct ClosureVisitor;
 
-impl<'d> serde::de::Visitor<'d> for ClosureVisitor<'_> {
+impl<'d> serde::de::Visitor<'d> for ClosureVisitor {
     type Value = MoveClosure;
 
     fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
@@ -377,13 +376,8 @@ mod serialization_tests {
         eprintln!("{:?}", blob);
         assert_eq!(
             value,
-            MoveValue::simple_deserialize(
-                &blob,
-                // The type layout is currently ignored by the serializer, so pass in some
-                // arbitrary one
-                &MoveTypeLayout::Function(MoveFunctionLayout(vec![], vec![], AbilitySet::EMPTY))
-            )
-            .expect("deserialization must succeed"),
+            MoveValue::simple_deserialize(&blob, &MoveTypeLayout::Function)
+                .expect("deserialization must succeed"),
             "deserialized value not equal to original one"
         );
     }
@@ -391,42 +385,6 @@ mod serialization_tests {
 
 //===========================================================================================
 
-impl fmt::Display for MoveFunctionLayout {
-    fn fmt(&self, f: &mut fmt::Formatter) -> std::fmt::Result {
-        let fmt_list = |l: &[MoveTypeLayout]| {
-            l.iter()
-                .map(|t| t.to_string())
-                .collect::<Vec<_>>()
-                .join(", ")
-        };
-        let MoveFunctionLayout(args, results, abilities) = self;
-        write!(
-            f,
-            "|{}|{}{}",
-            fmt_list(args),
-            fmt_list(results),
-            abilities.display_postfix()
-        )
-    }
-}
-
-impl TryInto<FunctionTag> for &MoveFunctionLayout {
-    type Error = anyhow::Error;
-
-    fn try_into(self) -> Result<FunctionTag, Self::Error> {
-        let into_list = |ts: &[MoveTypeLayout]| {
-            ts.iter()
-                .map(|t| t.try_into())
-                .collect::<Result<Vec<TypeTag>, _>>()
-        };
-        Ok(FunctionTag {
-            args: into_list(&self.0)?,
-            results: into_list(&self.1)?,
-            abilities: self.2,
-        })
-    }
-}
-
 impl fmt::Display for MoveClosure {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         let MoveClosure {
diff --git a/third_party/move/move-core/types/src/value.rs b/third_party/move/move-core/types/src/value.rs
index c1955d286d5..5265fd39b32 100644
--- a/third_party/move/move-core/types/src/value.rs
+++ b/third_party/move/move-core/types/src/value.rs
@@ -9,7 +9,7 @@
 
 use crate::{
     account_address::AccountAddress,
-    function::{ClosureVisitor, MoveClosure, MoveFunctionLayout},
+    function::{ClosureVisitor, MoveClosure},
     ident_str,
     identifier::Identifier,
     language_storage::{ModuleId, StructTag, TypeTag},
@@ -267,7 +267,7 @@ pub enum MoveTypeLayout {
 
     // Added in bytecode version v8
     #[serde(rename(serialize = "fun", deserialize = "fun"))]
-    Function(MoveFunctionLayout),
+    Function,
 }
 
 impl MoveTypeLayout {
@@ -276,17 +276,6 @@ impl MoveTypeLayout {
     pub fn is_compatible_with(&self, other: &Self) -> bool {
         use MoveTypeLayout::*;
         match (self, other) {
-            (
-                Function(MoveFunctionLayout(args1, res1, ab1)),
-                Function(MoveFunctionLayout(args2, res2, ab2)),
-            ) => {
-                // Notice that (currently) the function layout is not influencing
-                // serialization, but we anyway don't want it to diverge.
-                // Notice contra-variance for arguments.
-                Self::is_compatible_with_slice(args2, args1)
-                    && Self::is_compatible_with_slice(res1, res2)
-                    && ab1.is_subset(*ab2)
-            },
             (Vector(t1), Vector(t2)) => t1.is_compatible_with(t2),
             (Struct(s1), Struct(s2)) => s1.is_compatible_with(s2),
             // For all other cases, equality is used
@@ -623,8 +612,8 @@ impl<'d> serde::de::DeserializeSeed<'d> for &MoveTypeLayout {
             },
             MoveTypeLayout::Signer => Err(D::Error::custom("cannot deserialize signer")),
             MoveTypeLayout::Struct(ty) => Ok(MoveValue::Struct(ty.deserialize(deserializer)?)),
-            MoveTypeLayout::Function(fun) => Ok(MoveValue::Closure(Box::new(
-                deserializer.deserialize_seq(ClosureVisitor(fun))?,
+            MoveTypeLayout::Function => Ok(MoveValue::Closure(Box::new(
+                deserializer.deserialize_seq(ClosureVisitor)?,
             ))),
             MoveTypeLayout::Vector(layout) => Ok(MoveValue::Vector(
                 deserializer.deserialize_seq(VectorElementVisitor(layout))?,
@@ -952,7 +941,7 @@ impl fmt::Display for MoveTypeLayout {
             Address => write!(f, "address"),
             Vector(typ) => write!(f, "vector<{}>", typ),
             Struct(s) => fmt::Display::fmt(s, f),
-            Function(fun) => fmt::Display::fmt(fun, f),
+            Function => write!(f, "function"),
             Signer => write!(f, "signer"),
             // TODO[agg_v2](cleanup): consider printing the tag as well.
             Native(_, typ) => write!(f, "native<{}>", typ),
@@ -1020,7 +1009,12 @@ impl TryInto<TypeTag> for &MoveTypeLayout {
             MoveTypeLayout::Signer => TypeTag::Signer,
             MoveTypeLayout::Vector(v) => TypeTag::Vector(Box::new(v.as_ref().try_into()?)),
             MoveTypeLayout::Struct(v) => TypeTag::Struct(Box::new(v.try_into()?)),
-            MoveTypeLayout::Function(f) => TypeTag::Function(Box::new(f.try_into()?)),
+
+            // For function values, we cannot reconstruct the tag because we do not know the
+            // argument and return types.
+            MoveTypeLayout::Function => {
+                bail!("Function layout cannot be constructed from type tag")
+            },
 
             // Native layout variant is only used by MoveVM, and is irrelevant
             // for type tags which are used to key resources in the global state.
diff --git a/third_party/move/move-vm/runtime/src/storage/ty_layout_converter.rs b/third_party/move/move-vm/runtime/src/storage/ty_layout_converter.rs
index 7483f7d02e6..b767b6d5d2a 100644
--- a/third_party/move/move-vm/runtime/src/storage/ty_layout_converter.rs
+++ b/third_party/move/move-vm/runtime/src/storage/ty_layout_converter.rs
@@ -4,7 +4,6 @@
 use crate::{config::VMConfig, storage::ty_tag_converter::TypeTagConverter, ModuleStorage};
 use move_binary_format::errors::{PartialVMError, PartialVMResult};
 use move_core_types::{
-    function::MoveFunctionLayout,
     language_storage::StructTag,
     value::{IdentifierMappingKind, MoveFieldLayout, MoveStructLayout, MoveTypeLayout},
     vm_status::StatusCode,
@@ -149,32 +148,9 @@ pub(crate) trait LayoutConverterBase {
                 *count += 1;
                 self.struct_name_to_type_layout(*idx, ty_args, count, depth + 1)?
             },
-            Type::Function {
-                args,
-                results,
-                abilities,
-            } => {
+            Type::Function { .. } => {
                 *count += 1;
-                let mut identifier_mapping = false;
-                let mut to_list = |tys: &[Type]| {
-                    tys.iter()
-                        .map(|ety| {
-                            self.type_to_type_layout_impl(ety, count, depth + 1)
-                                .map(|(l, has)| {
-                                    identifier_mapping |= has;
-                                    l
-                                })
-                        })
-                        .collect::<PartialVMResult<Vec<_>>>()
-                };
-                (
-                    MoveTypeLayout::Function(MoveFunctionLayout(
-                        to_list(args)?,
-                        to_list(results)?,
-                        *abilities,
-                    )),
-                    identifier_mapping,
-                )
+                (MoveTypeLayout::Function, false)
             },
             Type::Reference(_) | Type::MutableReference(_) | Type::TyParam(_) => {
                 return Err(
@@ -311,22 +287,7 @@ pub(crate) trait LayoutConverterBase {
             Type::StructInstantiation { idx, ty_args, .. } => {
                 self.struct_name_to_fully_annotated_layout(*idx, ty_args, count, depth + 1)?
             },
-            Type::Function {
-                args,
-                results,
-                abilities,
-            } => {
-                let mut to_list = |tys: &[Type]| {
-                    tys.iter()
-                        .map(|ety| self.type_to_fully_annotated_layout_impl(ety, count, depth + 1))
-                        .collect::<PartialVMResult<Vec<_>>>()
-                };
-                MoveTypeLayout::Function(MoveFunctionLayout(
-                    to_list(args)?,
-                    to_list(results)?,
-                    *abilities,
-                ))
-            },
+            Type::Function { .. } => MoveTypeLayout::Function,
             Type::Reference(_) | Type::MutableReference(_) | Type::TyParam(_) => {
                 return Err(
                     PartialVMError::new(StatusCode::UNKNOWN_INVARIANT_VIOLATION_ERROR)
diff --git a/third_party/move/move-vm/types/src/values/function_values_impl.rs b/third_party/move/move-vm/types/src/values/function_values_impl.rs
index f1f1564f3fd..6c5d0e01743 100644
--- a/third_party/move/move-vm/types/src/values/function_values_impl.rs
+++ b/third_party/move/move-vm/types/src/values/function_values_impl.rs
@@ -5,7 +5,7 @@ use crate::values::{DeserializationSeed, SerializationReadyValue, VMValueCast, V
 use better_any::Tid;
 use move_binary_format::errors::{PartialVMError, PartialVMResult};
 use move_core_types::{
-    function::{ClosureMask, MoveFunctionLayout, FUNCTION_DATA_SERIALIZATION_FORMAT_V1},
+    function::{ClosureMask, FUNCTION_DATA_SERIALIZATION_FORMAT_V1},
     identifier::Identifier,
     language_storage::{ModuleId, TypeTag},
     value::MoveTypeLayout,
@@ -108,7 +108,7 @@ impl VMValueCast<Closure> for Value {
     }
 }
 
-impl serde::Serialize for SerializationReadyValue<'_, '_, '_, MoveFunctionLayout, Closure> {
+impl serde::Serialize for SerializationReadyValue<'_, '_, '_, (), Closure> {
     fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
         let Closure(fun, captured) = self.value;
         let fun_ext = self
@@ -136,11 +136,9 @@ impl serde::Serialize for SerializationReadyValue<'_, '_, '_, MoveFunctionLayout
     }
 }
 
-pub(crate) struct ClosureVisitor<'c, 'l>(
-    pub(crate) DeserializationSeed<'c, &'l MoveFunctionLayout>,
-);
+pub(crate) struct ClosureVisitor<'c>(pub(crate) DeserializationSeed<'c, ()>);
 
-impl<'d, 'c, 'l> serde::de::Visitor<'d> for ClosureVisitor<'c, 'l> {
+impl<'d, 'c> serde::de::Visitor<'d> for ClosureVisitor<'c> {
     type Value = Closure;
 
     fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
diff --git a/third_party/move/move-vm/types/src/values/serialization_tests.rs b/third_party/move/move-vm/types/src/values/serialization_tests.rs
index 7275f844537..b19cdc41cc9 100644
--- a/third_party/move/move-vm/types/src/values/serialization_tests.rs
+++ b/third_party/move/move-vm/types/src/values/serialization_tests.rs
@@ -16,9 +16,7 @@ mod tests {
     use move_core_types::{
         ability::AbilitySet,
         account_address::AccountAddress,
-        function::{
-            ClosureMask, MoveClosure, MoveFunctionLayout, FUNCTION_DATA_SERIALIZATION_FORMAT_V1,
-        },
+        function::{ClosureMask, MoveClosure, FUNCTION_DATA_SERIALIZATION_FORMAT_V1},
         identifier::Identifier,
         language_storage::{FunctionTag, ModuleId, StructTag, TypeTag},
         u256,
@@ -208,12 +206,7 @@ mod tests {
     // Closures
 
     fn make_fun_layout() -> MoveTypeLayout {
-        // Currently, content of layout doesn't influence parsing, so can set anything here
-        MoveTypeLayout::Function(MoveFunctionLayout(
-            vec![],
-            vec![],
-            AbilitySet::PUBLIC_FUNCTIONS,
-        ))
+        MoveTypeLayout::Function
     }
 
     fn make_type_args() -> Vec<TypeTag> {
diff --git a/third_party/move/move-vm/types/src/values/values_impl.rs b/third_party/move/move-vm/types/src/values/values_impl.rs
index ec0140bac6f..1ddfd9940c2 100644
--- a/third_party/move/move-vm/types/src/values/values_impl.rs
+++ b/third_party/move/move-vm/types/src/values/values_impl.rs
@@ -3770,9 +3770,9 @@ impl serde::Serialize for SerializationReadyValue<'_, '_, '_, MoveTypeLayout, Va
             },
 
             // Functions.
-            (L::Function(fun_layout), ValueImpl::ClosureValue(clos)) => SerializationReadyValue {
+            (L::Function, ValueImpl::ClosureValue(clos)) => SerializationReadyValue {
                 ctx: self.ctx,
-                layout: fun_layout,
+                layout: &(),
                 value: clos,
             }
             .serialize(serializer),
@@ -4042,10 +4042,10 @@ impl<'d> serde::de::DeserializeSeed<'d> for DeserializationSeed<'_, &MoveTypeLay
             }),
 
             // Functions
-            L::Function(fun_layout) => {
+            L::Function => {
                 let seed = DeserializationSeed {
                     ctx: self.ctx,
-                    layout: fun_layout,
+                    layout: (),
                 };
                 let closure = deserializer.deserialize_seq(ClosureVisitor(seed))?;
                 Ok(Value(ValueImpl::ClosureValue(closure)))
@@ -4710,7 +4710,7 @@ pub mod prop {
                 .prop_map(move |vals| Value::struct_(Struct::pack(vals)))
                 .boxed(),
 
-            L::Function(_function_layout) => {
+            L::Function => {
                 // TODO(#15664): not clear how to generate closure values, we'd need
                 //   some test functions for this, and generate `AbstractFunction` impls.
                 //   As we do not generate function layouts in the first place, we can bail
diff --git a/third_party/move/tools/move-bytecode-utils/src/layout.rs b/third_party/move/tools/move-bytecode-utils/src/layout.rs
index 00235b3b32b..2f5688d4f25 100644
--- a/third_party/move/tools/move-bytecode-utils/src/layout.rs
+++ b/third_party/move/tools/move-bytecode-utils/src/layout.rs
@@ -14,7 +14,6 @@ use move_binary_format::{
 };
 use move_core_types::{
     account_address::AccountAddress,
-    function::MoveFunctionLayout,
     identifier::{IdentStr, Identifier},
     language_storage::{ModuleId, StructTag, TypeTag},
     value::{MoveFieldLayout, MoveStructLayout, MoveTypeLayout},
@@ -375,18 +374,7 @@ impl TypeLayoutBuilder {
                 compiled_module_view,
                 layout_type,
             )?),
-            Function(f) => {
-                let build_list = |ts: &[TypeTag]| {
-                    ts.iter()
-                        .map(|t| Self::build(t, compiled_module_view, layout_type))
-                        .collect::<anyhow::Result<Vec<_>>>()
-                };
-                MoveTypeLayout::Function(MoveFunctionLayout(
-                    build_list(&f.args)?,
-                    build_list(&f.results)?,
-                    f.abilities,
-                ))
-            },
+            Function(_) => MoveTypeLayout::Function,
         })
     }
 
diff --git a/third_party/move/tools/move-resource-viewer/src/fat_type.rs b/third_party/move/tools/move-resource-viewer/src/fat_type.rs
index 283da96db40..4e5577e7e97 100644
--- a/third_party/move/tools/move-resource-viewer/src/fat_type.rs
+++ b/third_party/move/tools/move-resource-viewer/src/fat_type.rs
@@ -8,7 +8,6 @@ use move_binary_format::errors::{PartialVMError, PartialVMResult};
 use move_core_types::{
     ability::AbilitySet,
     account_address::AccountAddress,
-    function::MoveFunctionLayout,
     identifier::Identifier,
     language_storage::{FunctionTag, StructTag, TypeTag},
     value::{MoveStructLayout, MoveTypeLayout},
@@ -367,14 +366,8 @@ impl FatType {
                     .map(|tys| Self::from_layout_slice(tys, limit))
                     .collect::<PartialVMResult<Vec<Vec<_>>>>()?,
             ),
-            Function(MoveFunctionLayout(args, results, abilities)) => {
-                FatType::Function(Box::new(FatFunctionType {
-                    args: Self::from_layout_slice(args, limit)?,
-                    results: Self::from_layout_slice(results, limit)?,
-                    abilities: *abilities,
-                }))
-            },
-            Native(..) | Struct(_) => {
+            // TODO(#15664): get rid of fat type to support captured functions.
+            Native(..) | Struct(_) | Function => {
                 return Err(PartialVMError::new_invariant_violation(format!(
                     "cannot derive fat type for {:?}",
                     layout
@@ -458,18 +451,6 @@ impl TryInto<MoveStructLayout> for &FatStructType {
     }
 }
 
-impl TryInto<MoveFunctionLayout> for &FatFunctionType {
-    type Error = PartialVMError;
-
-    fn try_into(self) -> Result<MoveFunctionLayout, Self::Error> {
-        Ok(MoveFunctionLayout(
-            into_types(self.args.iter())?,
-            into_types(self.results.iter())?,
-            self.abilities,
-        ))
-    }
-}
-
 fn into_types<'a>(
     types: impl Iterator<Item = &'a FatType>,
 ) -> PartialVMResult<Vec<MoveTypeLayout>> {
@@ -498,7 +479,7 @@ impl TryInto<MoveTypeLayout> for &FatType {
             FatType::Bool => MoveTypeLayout::Bool,
             FatType::Vector(v) => MoveTypeLayout::Vector(Box::new(v.as_ref().try_into()?)),
             FatType::Struct(s) => MoveTypeLayout::Struct(s.as_ref().try_into()?),
-            FatType::Function(f) => MoveTypeLayout::Function(f.as_ref().try_into()?),
+            FatType::Function(_) => MoveTypeLayout::Function,
             FatType::Runtime(tys) => {
                 MoveTypeLayout::Struct(MoveStructLayout::Runtime(slice_into(tys)?))
             },


Reviews:
user: login: wrwg
body: Great fix. Making it just a tag was the intention after the first iteration, but fall through the cracks.
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-05-24T05:42:53Z
user: login: vineethk
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-05-28T19:11:23Z
user: login: georgemitenkov
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-29T08:25:04Z
user: login: georgemitenkov
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-29T08:29:51Z
user: login: georgemitenkov
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-29T08:32:09Z
user: login: zi0Black
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-30T09:30:07Z
user: login: georgemitenkov
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-30T10:12:52Z

Review comments:
diff_hunk: @@ -367,14 +366,7 @@ impl FatType {
                     .map(|tys| Self::from_layout_slice(tys, limit))
                     .collect::<PartialVMResult<Vec<Vec<_>>>>()?,
             ),
-            Function(MoveFunctionLayout(args, results, abilities)) => {
-                FatType::Function(Box::new(FatFunctionType {
-                    args: Self::from_layout_slice(args, limit)?,
-                    results: Self::from_layout_slice(results, limit)?,
-                    abilities: *abilities,
-                }))
-            },
-            Native(..) | Struct(_) => {
+            Native(..) | Struct(_) | Function => {
path: third_party/move/tools/move-resource-viewer/src/fat_type.rs
user: login: vineethk
body: On a cursory look at the call sites of this function, it looks like one could call this function with `layout` being `Function`? Could you explain the reasoning behind why this change is okay?
created_at: 2025-05-28T16:33:30Z
author_association: CONTRIBUTOR
diff_hunk: @@ -253,7 +247,6 @@ mod tests {
 
     #[test]
     fn closure_round_trip_move_value_good() {
path: third_party/move/move-vm/types/src/values/serialization_tests.rs
user: login: vineethk
body: Why these changes in the tests? They can be left as is?
created_at: 2025-05-28T16:45:33Z
author_association: CONTRIBUTOR
diff_hunk: @@ -209,11 +207,7 @@ mod tests {
 
     fn make_fun_layout() -> MoveTypeLayout {
         // Currently, content of layout doesn't influence parsing, so can set anything here
path: third_party/move/move-vm/types/src/values/serialization_tests.rs
user: login: vineethk
body: Maybe this comment does not apply anymore (not sure what is being "set")?
created_at: 2025-05-28T16:46:10Z
author_association: CONTRIBUTOR
diff_hunk: @@ -209,11 +207,7 @@ mod tests {
 
     fn make_fun_layout() -> MoveTypeLayout {
         // Currently, content of layout doesn't influence parsing, so can set anything here
path: third_party/move/move-vm/types/src/values/serialization_tests.rs
user: login: georgemitenkov
body: Indeed, removed
created_at: 2025-05-29T08:25:04Z
author_association: CONTRIBUTOR
diff_hunk: @@ -367,14 +366,7 @@ impl FatType {
                     .map(|tys| Self::from_layout_slice(tys, limit))
                     .collect::<PartialVMResult<Vec<Vec<_>>>>()?,
             ),
-            Function(MoveFunctionLayout(args, results, abilities)) => {
-                FatType::Function(Box::new(FatFunctionType {
-                    args: Self::from_layout_slice(args, limit)?,
-                    results: Self::from_layout_slice(results, limit)?,
-                    abilities: *abilities,
-                }))
-            },
-            Native(..) | Struct(_) => {
+            Native(..) | Struct(_) | Function => {
path: third_party/move/tools/move-resource-viewer/src/fat_type.rs
user: login: georgemitenkov
body: Yes, if closure captured a closure this can fail. I would argue we need to get rid of this fat type altogether, to make sure this is properly fixed. Discussed with @wrwg in person that we go over this code again and it would be ideal to get rid of fat type. The current usage is `TypeTag` -> `FatType` --> `MoveTypeLayout` but all we need is layout really, so fat type is useless legacy. I have some WIP to get rid of it (will put a PR some time next week)
created_at: 2025-05-29T08:29:50Z
author_association: CONTRIBUTOR
diff_hunk: @@ -367,14 +366,7 @@ impl FatType {
                     .map(|tys| Self::from_layout_slice(tys, limit))
                     .collect::<PartialVMResult<Vec<Vec<_>>>>()?,
             ),
-            Function(MoveFunctionLayout(args, results, abilities)) => {
-                FatType::Function(Box::new(FatFunctionType {
-                    args: Self::from_layout_slice(args, limit)?,
-                    results: Self::from_layout_slice(results, limit)?,
-                    abilities: *abilities,
-                }))
-            },
-            Native(..) | Struct(_) => {
+            Native(..) | Struct(_) | Function => {
path: third_party/move/tools/move-resource-viewer/src/fat_type.rs
user: login: georgemitenkov
body: Added a TODO here so it is tracked, in any case.
created_at: 2025-05-29T08:32:09Z
author_association: CONTRIBUTOR
diff_hunk: @@ -367,14 +366,7 @@ impl FatType {
                     .map(|tys| Self::from_layout_slice(tys, limit))
                     .collect::<PartialVMResult<Vec<Vec<_>>>>()?,
             ),
-            Function(MoveFunctionLayout(args, results, abilities)) => {
-                FatType::Function(Box::new(FatFunctionType {
-                    args: Self::from_layout_slice(args, limit)?,
-                    results: Self::from_layout_slice(results, limit)?,
-                    abilities: *abilities,
-                }))
-            },
-            Native(..) | Struct(_) => {
+            Native(..) | Struct(_) | Function => {
path: third_party/move/tools/move-resource-viewer/src/fat_type.rs
user: login: zi0Black
body: Not a fan TODOs those remain there forever. Can we have a GitHub issues or a task somewhere.
created_at: 2025-05-30T09:30:07Z
author_association: CONTRIBUTOR
diff_hunk: @@ -367,14 +366,7 @@ impl FatType {
                     .map(|tys| Self::from_layout_slice(tys, limit))
                     .collect::<PartialVMResult<Vec<Vec<_>>>>()?,
             ),
-            Function(MoveFunctionLayout(args, results, abilities)) => {
-                FatType::Function(Box::new(FatFunctionType {
-                    args: Self::from_layout_slice(args, limit)?,
-                    results: Self::from_layout_slice(results, limit)?,
-                    abilities: *abilities,
-                }))
-            },
-            Native(..) | Struct(_) => {
+            Native(..) | Struct(_) | Function => {
path: third_party/move/tools/move-resource-viewer/src/fat_type.rs
user: login: georgemitenkov
body: Landed because of auto-merge :(( created the issue still https://github.com/aptos-labs/aptos-core/issues/16727
created_at: 2025-05-30T10:12:52Z
author_association: CONTRIBUTOR

PR comments:
user: login: georgemitenkov
created_at: 2025-05-23T09:03:59Z
body: * **#16714** <a href="https://app.graphite.dev/github/pr/aptos-labs/aptos-core/16714?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a>
* **#16682** <a href="https://app.graphite.dev/github/pr/aptos-labs/aptos-core/16682?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a> 👈 <a href="https://app.graphite.dev/github/pr/aptos-labs/aptos-core/16682?utm_source=stack-comment-view-in-graphite" target="_blank">(View in Graphite)</a>
* **#16656** <a href="https://app.graphite.dev/github/pr/aptos-labs/aptos-core/16656?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a>
* `main`


<h2></h2>

This stack of pull requests is managed by <a href="https://graphite.dev?utm-source=stack-comment"><b>Graphite</b></a>. Learn more about <a href="https://stacking.dev/?utm_source=stack-comment">stacking</a>.
<!-- Current dependencies on/for this PR: -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-05-29T11:06:16Z
body: ### Forge is running suite `compat` on `6ff41a58d52e09e11cf6d88c770b376f1108749d` ==> `968bb60e68d464106456d250ab829a90aae265a7`
* [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-compat-pr-16682&var-metrics_source=All&var-chain_name=forge-0&refresh=10s&from=now-15m&to=now)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-compat-pr-16682%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-compat-pr-16682%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16682%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16682%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22quickRange%22%3A%20%2230m%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16682&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16682&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/15319807403)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-compat -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-05-29T11:06:17Z
body: ### Forge is running suite `framework_upgrade` on `6ff41a58d52e09e11cf6d88c770b376f1108749d` ==> `968bb60e68d464106456d250ab829a90aae265a7`
* [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-framework-upgrade-pr-16682&var-metrics_source=All&var-chain_name=forge-1&refresh=10s&from=now-15m&to=now)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-framework-upgrade-pr-16682%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-framework-upgrade-pr-16682%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16682%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16682%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22quickRange%22%3A%20%2230m%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16682&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16682&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/15319807403)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-framework-upgrade -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-05-29T11:06:18Z
body: ### Forge is running suite `realistic_env_max_load` on `968bb60e68d464106456d250ab829a90aae265a7`
* [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-e2e-pr-16682&var-metrics_source=All&var-chain_name=forge-1&refresh=10s&from=now-15m&to=now)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-e2e-pr-16682%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-e2e-pr-16682%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16682%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16682%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22quickRange%22%3A%20%2230m%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16682&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16682&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/15319807403)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-e2e -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-05-29T11:15:36Z
body: ### :white_check_mark: Forge suite `compat` success on `6ff41a58d52e09e11cf6d88c770b376f1108749d` ==> `968bb60e68d464106456d250ab829a90aae265a7`
```
Forge report malformed: Expecting property name enclosed in double quotes: line 2 column 1 (char 2)
'{\n[2025-05-29T11:15:32Z INFO  aptos_forge::report] Test Ok\n  "metrics": [\n    {\n      "test_name": "compatibility::simple-validator-upgrade::liveness-check",\n      "metric": "submitted_txn",\n      "value": 310120.0\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::liveness-check",\n      "metric": "expired_txn",\n      "value": 0.0\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::liveness-check",\n      "metric": "avg_tps",\n      "value": 9368.731533483431\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::liveness-check",\n      "metric": "avg_latency",\n      "value": 3625.2632045659743\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::liveness-check",\n      "metric": "p50_latency",\n      "value": 3800.0\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::liveness-check",\n      "metric": "p90_latency",\n      "value": 4300.0\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::liveness-check",\n      "metric": "p99_latency",\n      "value": 4700.0\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::single-validator-upgrade",\n      "metric": "submitted_txn",\n      "value": 104420.0\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::single-validator-upgrade",\n      "metric": "expired_txn",\n      "value": 0.0\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::single-validator-upgrade",\n      "metric": "avg_tps",\n      "value": 2783.7978994341192\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::single-validator-upgrade",\n      "metric": "avg_latency",\n      "value": 11960.276307220838\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::single-validator-upgrade",\n      "metric": "p50_latency",\n      "value": 13600.0\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::single-validator-upgrade",\n      "metric": "p90_latency",\n      "value": 14000.0\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::single-validator-upgrade",\n      "metric": "p99_latency",\n      "value": 14300.0\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::half-validator-upgrade",\n      "metric": "submitted_txn",\n      "value": 103560.0\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::half-validator-upgrade",\n      "metric": "expired_txn",\n      "value": 0.0\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::half-validator-upgrade",\n      "metric": "avg_tps",\n      "value": 2704.326237406142\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::half-validator-upgrade",\n      "metric": "avg_latency",\n      "value": 12256.293984163769\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::half-validator-upgrade",\n      "metric": "p50_latency",\n      "value": 13300.0\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::half-validator-upgrade",\n      "metric": "p90_latency",\n      "value": 14700.0\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::half-validator-upgrade",\n      "metric": "p99_latency",\n      "value": 14800.0\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::rest-validator-upgrade",\n      "metric": "submitted_txn",\n      "value": 167460.0\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::rest-validator-upgrade",\n      "metric": "expired_txn",\n      "value": 0.0\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::rest-validator-upgrade",\n      "metric": "avg_tps",\n      "value": 4752.609071916236\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::rest-validator-upgrade",\n      "metric": "avg_latency",\n      "value": 7174.046088618177\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::rest-validator-upgrade",\n      "metric": "p50_latency",\n      "value": 7400.0\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::rest-validator-upgrade",\n      "metric": "p90_latency",\n      "value": 8900.0\n    },\n    {\n      "test_name": "compatibility::simple-validator-upgrade::rest-validator-upgrade",\n      "metric": "p99_latency",\n      "value": 9100.0\n    }\n  ],\n  "text": "Compatibility test results for 6ff41a58d52e09e11cf6d88c770b376f1108749d ==> 968bb60e68d464106456d250ab829a90aae265a7 (PR)\\n1. Check liveness of validators at old version: 6ff41a58d52e09e11cf6d88c770b376f1108749d\\ncompatibility::simple-validator-upgrade::liveness-check : committed: 9368.73 txn/s, latency: 3625.26 ms, (p50: 3800 ms, p70: 4000, p90: 4300 ms, p99: 4700 ms), latency samples: 310120\\n2. Upgrading first Validator to new version: 968bb60e68d464106456d250ab829a90aae265a7\\ncompatibility::simple-validator-upgrade::single-validator-upgrade : committed: 2783.80 txn/s, latency: 11960.28 ms, (p50: 13600 ms, p70: 13900, p90: 14000 ms, p99: 14300 ms), latency samples: 104420\\n3. Upgrading rest of first batch to new version: 968bb60e68d464106456d250ab829a90aae265a7\\ncompatibility::simple-validator-upgrade::half-validator-upgrade : committed: 2704.33 txn/s, latency: 12256.29 ms, (p50: 13300 ms, p70: 14300, p90: 14700 ms, p99: 14800 ms), latency samples: 103560\\n4. upgrading second batch to new version: 968bb60e68d464106456d250ab829a90aae265a7\\ncompatibility::simple-validator-upgrade::rest-validator-upgrade : committed: 4752.61 txn/s, latency: 7174.05 ms, (p50: 7400 ms, p70: 8200, p90: 8900 ms, p99: 9100 ms), latency samples: 167460\\n5. check swarm health\\nCompatibility test for 6ff41a58d52e09e11cf6d88c770b376f1108749d ==> 968bb60e68d464106456d250ab829a90aae265a7 passed\\nTest Ok"\n}'
Trailing Log Lines:
2. Upgrading first Validator to new version: 968bb60e68d464106456d250ab829a90aae265a7
compatibility::simple-validator-upgrade::single-validator-upgrade : committed: 2783.80 txn/s, latency: 11960.28 ms, (p50: 13600 ms, p70: 13900, p90: 14000 ms, p99: 14300 ms), latency samples: 104420
3. Upgrading rest of first batch to new version: 968bb60e68d464106456d250ab829a90aae265a7
compatibility::simple-validator-upgrade::half-validator-upgrade : committed: 2704.33 txn/s, latency: 12256.29 ms, (p50: 13300 ms, p70: 14300, p90: 14700 ms, p99: 14800 ms), latency samples: 103560
4. upgrading second batch to new version: 968bb60e68d464106456d250ab829a90aae265a7
compatibility::simple-validator-upgrade::rest-validator-upgrade : committed: 4752.61 txn/s, latency: 7174.05 ms, (p50: 7400 ms, p70: 8200, p90: 8900 ms, p99: 9100 ms), latency samples: 167460
5. check swarm health
Compatibility test for 6ff41a58d52e09e11cf6d88c770b376f1108749d ==> 968bb60e68d464106456d250ab829a90aae265a7 passed
Test Ok

=== BEGIN JUNIT ===
<?xml version="1.0" encoding="UTF-8"?>
<testsuites name="forge" tests="1" failures="0" errors="0" uuid="3e87a5c7-fed7-4282-8c3b-c9b7cf93ea23">
    <testsuite name="compat" tests="1" disabled="0" errors="0" failures="0">
        <testcase name="compatibility::simple-validator-upgrade">
        </testcase>
    </testsuite>
</testsuites>
=== END JUNIT ===
[2025-05-29T11:15:32Z INFO  aptos_forge::backend::k8s::cluster_helper] Deleting namespace forge-compat-pr-16682: Some(NamespaceStatus { conditions: None, phase: Some("Terminating") })
[2025-05-29T11:15:32Z INFO  aptos_forge::backend::k8s::cluster_helper] aptos-node resources for Forge removed in namespace: forge-compat-pr-16682

test result: ok. 1 passed; 0 failed; 0 filtered out

Debugging output:
NAME                                         READY   STATUS      RESTARTS   AGE
aptos-node-0-validator-0                     1/1     Running     0          4m29s
aptos-node-1-validator-0                     1/1     Running     0          6m15s
aptos-node-2-validator-0                     1/1     Running     0          2m35s
aptos-node-3-validator-0                     1/1     Running     0          104s
forge-testnet-deployer-j4sf7                 0/1     Completed   0          8m54s
genesis-aptos-genesis-eforge2531142e-jl9xm   0/1     Completed   0          8m47s

```
* [Grafana dashboard](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-compat-pr-16682&var-metrics_source=All&var-chain_name=forge-0&from=1748516783000&to=1748517334000)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-compat-pr-16682%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-compat-pr-16682%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1748516783000&end=1748517334000)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16682%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16682%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22startTime%22%3A%20%222025-05-29T11%3A06%3A23.905357%2B00%3A00%22%2C%20%22endTime%22%3A%20%222025-05-29T11%3A15%3A34.752871%2B00%3A00%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1748516783&until=1748517334&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16682&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1748516783&until=1748517334&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16682&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/15319807403)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-compat -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-05-29T11:17:45Z
body: ### :white_check_mark: Forge suite `realistic_env_max_load` success on `968bb60e68d464106456d250ab829a90aae265a7`
```
two traffics test: inner traffic : committed: 12537.77 txn/s, submitted: 12537.83 txn/s, expired: 0.05 txn/s, latency: 3165.74 ms, (p50: 3000 ms, p70: 3300, p90: 3700 ms, p99: 5400 ms), latency samples: 4767120
two traffics test : committed: 99.98 txn/s, latency: 2048.96 ms, (p50: 1500 ms, p70: 2600, p90: 3900 ms, p99: 6200 ms), latency samples: 1800
Latency breakdown for phase 0: ["MempoolToBlockCreation: max: 2.049, avg: 1.657", "ConsensusProposalToOrdered: max: 0.183, avg: 0.177", "ConsensusOrderedToCommit: max: 0.369, avg: 0.316", "ConsensusProposalToCommit: max: 0.544, avg: 0.493"]
Max non-epoch-change gap was: 0 rounds at version 0 (avg 0.00) [limit 4], 0.44s no progress at version 5457508 (avg 0.13s) [limit 15].
Max epoch-change gap was: 0 rounds at version 0 (avg 0.00) [limit 4], 0.54s no progress at version 2718139 (avg 0.54s) [limit 16].
Test Ok
```
* [Grafana dashboard](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-e2e-pr-16682&var-metrics_source=All&var-chain_name=forge-1&from=1748516784000&to=1748517463000)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-e2e-pr-16682%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-e2e-pr-16682%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1748516784000&end=1748517463000)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16682%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16682%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22startTime%22%3A%20%222025-05-29T11%3A06%3A24.546528%2B00%3A00%22%2C%20%22endTime%22%3A%20%222025-05-29T11%3A17%3A43.873605%2B00%3A00%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1748516784&until=1748517463&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16682&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1748516784&until=1748517463&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16682&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/15319807403)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-e2e -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-05-29T11:19:19Z
body: ### :white_check_mark: Forge suite `framework_upgrade` success on `6ff41a58d52e09e11cf6d88c770b376f1108749d` ==> `968bb60e68d464106456d250ab829a90aae265a7`
```
Compatibility test results for 6ff41a58d52e09e11cf6d88c770b376f1108749d ==> 968bb60e68d464106456d250ab829a90aae265a7 (PR)
Upgrade the nodes to version: 968bb60e68d464106456d250ab829a90aae265a7
framework_upgrade::framework-upgrade::full-framework-upgrade : committed: 999.90 txn/s, submitted: 1003.64 txn/s, failed submission: 3.74 txn/s, expired: 3.74 txn/s, latency: 3014.65 ms, (p50: 1800 ms, p70: 2100, p90: 11700 ms, p99: 13700 ms), latency samples: 90880
framework_upgrade::framework-upgrade::full-framework-upgrade : committed: 1719.17 txn/s, submitted: 1724.48 txn/s, failed submission: 5.31 txn/s, expired: 5.31 txn/s, latency: 1761.86 ms, (p50: 1500 ms, p70: 2100, p90: 2700 ms, p99: 3600 ms), latency samples: 155481
5. check swarm health
Compatibility test for 6ff41a58d52e09e11cf6d88c770b376f1108749d ==> 968bb60e68d464106456d250ab829a90aae265a7 passed
Upgrade the remaining nodes to version: 968bb60e68d464106456d250ab829a90aae265a7
framework_upgrade::framework-upgrade::full-framework-upgrade : committed: 1545.47 txn/s, submitted: 1550.02 txn/s, failed submission: 4.54 txn/s, expired: 4.54 txn/s, latency: 1920.58 ms, (p50: 1700 ms, p70: 2000, p90: 3300 ms, p99: 4600 ms), latency samples: 136022
Test Ok
```
* [Grafana dashboard](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-framework-upgrade-pr-16682&var-metrics_source=All&var-chain_name=forge-1&from=1748516784000&to=1748517557000)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-framework-upgrade-pr-16682%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-framework-upgrade-pr-16682%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1748516784000&end=1748517557000)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16682%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16682%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22startTime%22%3A%20%222025-05-29T11%3A06%3A24.221492%2B00%3A00%22%2C%20%22endTime%22%3A%20%222025-05-29T11%3A19%3A17.982306%2B00%3A00%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1748516784&until=1748517557&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16682&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1748516784&until=1748517557&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16682&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/15319807403)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-framework-upgrade -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-05-29T12:17:27Z
body: ### Forge is running suite `realistic_env_max_load` on `968bb60e68d464106456d250ab829a90aae265a7`
* [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-e2e-pr-16682&var-metrics_source=All&var-chain_name=forge-1&refresh=10s&from=now-15m&to=now)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-e2e-pr-16682%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-e2e-pr-16682%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16682%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16682%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22quickRange%22%3A%20%2230m%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16682&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16682&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/15323349930)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-e2e -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-05-29T12:17:27Z
body: ### Forge is running suite `compat` on `6ff41a58d52e09e11cf6d88c770b376f1108749d` ==> `968bb60e68d464106456d250ab829a90aae265a7`
* [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-compat-pr-16682&var-metrics_source=All&var-chain_name=forge-0&refresh=10s&from=now-15m&to=now)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-compat-pr-16682%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-compat-pr-16682%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16682%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16682%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22quickRange%22%3A%20%2230m%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16682&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16682&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/15323349930)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-compat -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-05-29T12:17:33Z
body: ### Forge is running suite `framework_upgrade` on `6ff41a58d52e09e11cf6d88c770b376f1108749d` ==> `968bb60e68d464106456d250ab829a90aae265a7`
* [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-framework-upgrade-pr-16682&var-metrics_source=All&var-chain_name=forge-1&refresh=10s&from=now-15m&to=now)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-framework-upgrade-pr-16682%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-framework-upgrade-pr-16682%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16682%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16682%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22quickRange%22%3A%20%2230m%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16682&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16682&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/15323349930)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-framework-upgrade -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-05-29T12:26:07Z
body: ### :white_check_mark: Forge suite `compat` success on `6ff41a58d52e09e11cf6d88c770b376f1108749d` ==> `968bb60e68d464106456d250ab829a90aae265a7`
```
Compatibility test results for 6ff41a58d52e09e11cf6d88c770b376f1108749d ==> 968bb60e68d464106456d250ab829a90aae265a7 (PR)
1. Check liveness of validators at old version: 6ff41a58d52e09e11cf6d88c770b376f1108749d
compatibility::simple-validator-upgrade::liveness-check : committed: 8990.71 txn/s, latency: 3773.08 ms, (p50: 4000 ms, p70: 4200, p90: 4700 ms, p99: 5300 ms), latency samples: 299500
2. Upgrading first Validator to new version: 968bb60e68d464106456d250ab829a90aae265a7
compatibility::simple-validator-upgrade::single-validator-upgrade : committed: 2707.89 txn/s, latency: 12281.93 ms, (p50: 13900 ms, p70: 14200, p90: 14500 ms, p99: 14600 ms), latency samples: 101280
3. Upgrading rest of first batch to new version: 968bb60e68d464106456d250ab829a90aae265a7
compatibility::simple-validator-upgrade::half-validator-upgrade : committed: 2599.73 txn/s, latency: 12850.40 ms, (p50: 14400 ms, p70: 14800, p90: 15200 ms, p99: 15300 ms), latency samples: 97600
4. upgrading second batch to new version: 968bb60e68d464106456d250ab829a90aae265a7
compatibility::simple-validator-upgrade::rest-validator-upgrade : committed: 4806.67 txn/s, latency: 7032.05 ms, (p50: 7300 ms, p70: 7900, p90: 8900 ms, p99: 9000 ms), latency samples: 169820
5. check swarm health
Compatibility test for 6ff41a58d52e09e11cf6d88c770b376f1108749d ==> 968bb60e68d464106456d250ab829a90aae265a7 passed
Test Ok
```
* [Grafana dashboard](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-compat-pr-16682&var-metrics_source=All&var-chain_name=forge-0&from=1748521053000&to=1748521566000)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-compat-pr-16682%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-compat-pr-16682%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1748521053000&end=1748521566000)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16682%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16682%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22startTime%22%3A%20%222025-05-29T12%3A17%3A33.799944%2B00%3A00%22%2C%20%22endTime%22%3A%20%222025-05-29T12%3A26%3A06.550009%2B00%3A00%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1748521053&until=1748521566&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16682&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1748521053&until=1748521566&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16682&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/15323349930)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-compat -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-05-29T12:28:39Z
body: ### :white_check_mark: Forge suite `realistic_env_max_load` success on `968bb60e68d464106456d250ab829a90aae265a7`
```
two traffics test: inner traffic : committed: 13416.96 txn/s, latency: 2964.08 ms, (p50: 2700 ms, p70: 2800, p90: 3300 ms, p99: 8400 ms), latency samples: 5101500
two traffics test : committed: 99.99 txn/s, latency: 2447.47 ms, (p50: 2300 ms, p70: 2600, p90: 2800 ms, p99: 10400 ms), latency samples: 1700
Latency breakdown for phase 0: ["MempoolToBlockCreation: max: 1.254, avg: 0.557", "ConsensusProposalToOrdered: max: 0.173, avg: 0.170", "ConsensusOrderedToCommit: max: 0.381, avg: 0.294", "ConsensusProposalToCommit: max: 0.550, avg: 0.464"]
Max non-epoch-change gap was: 0 rounds at version 0 (avg 0.00) [limit 4], 0.64s no progress at version 78147 (avg 0.12s) [limit 15].
Max epoch-change gap was: 0 rounds at version 0 (avg 0.00) [limit 4], 0.65s no progress at version 2926141 (avg 0.65s) [limit 16].
Test Ok
```
* [Grafana dashboard](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-e2e-pr-16682&var-metrics_source=All&var-chain_name=forge-1&from=1748521053000&to=1748521717000)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-e2e-pr-16682%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-e2e-pr-16682%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1748521053000&end=1748521717000)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16682%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16682%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22startTime%22%3A%20%222025-05-29T12%3A17%3A33.580871%2B00%3A00%22%2C%20%22endTime%22%3A%20%222025-05-29T12%3A28%3A37.611985%2B00%3A00%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1748521053&until=1748521717&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16682&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1748521053&until=1748521717&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16682&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/15323349930)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-e2e -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-05-29T12:30:59Z
body: ### :white_check_mark: Forge suite `framework_upgrade` success on `6ff41a58d52e09e11cf6d88c770b376f1108749d` ==> `968bb60e68d464106456d250ab829a90aae265a7`
```
Compatibility test results for 6ff41a58d52e09e11cf6d88c770b376f1108749d ==> 968bb60e68d464106456d250ab829a90aae265a7 (PR)
Upgrade the nodes to version: 968bb60e68d464106456d250ab829a90aae265a7
framework_upgrade::framework-upgrade::full-framework-upgrade : committed: 1623.47 txn/s, submitted: 1630.74 txn/s, failed submission: 7.28 txn/s, expired: 7.28 txn/s, latency: 1787.07 ms, (p50: 1500 ms, p70: 1900, p90: 3000 ms, p99: 4000 ms), latency samples: 147261
framework_upgrade::framework-upgrade::full-framework-upgrade : committed: 1672.91 txn/s, submitted: 1677.99 txn/s, failed submission: 5.08 txn/s, expired: 5.08 txn/s, latency: 1779.26 ms, (p50: 1700 ms, p70: 2000, p90: 2700 ms, p99: 3700 ms), latency samples: 151561
5. check swarm health
Compatibility test for 6ff41a58d52e09e11cf6d88c770b376f1108749d ==> 968bb60e68d464106456d250ab829a90aae265a7 passed
Upgrade the remaining nodes to version: 968bb60e68d464106456d250ab829a90aae265a7
framework_upgrade::framework-upgrade::full-framework-upgrade : committed: 1534.19 txn/s, submitted: 1538.94 txn/s, failed submission: 4.76 txn/s, expired: 4.76 txn/s, latency: 1874.32 ms, (p50: 1700 ms, p70: 2100, p90: 2900 ms, p99: 4200 ms), latency samples: 135460
Test Ok
```
* [Grafana dashboard](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-framework-upgrade-pr-16682&var-metrics_source=All&var-chain_name=forge-1&from=1748521059000&to=1748521858000)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-framework-upgrade-pr-16682%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-framework-upgrade-pr-16682%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1748521059000&end=1748521858000)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16682%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16682%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22startTime%22%3A%20%222025-05-29T12%3A17%3A39.699823%2B00%3A00%22%2C%20%22endTime%22%3A%20%222025-05-29T12%3A30%3A58.228916%2B00%3A00%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1748521059&until=1748521858&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16682&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1748521059&until=1748521858&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16682&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/15323349930)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-framework-upgrade -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-05-30T09:13:24Z
body: ### Forge is running suite `framework_upgrade` on `ead1a2d60bb9fe8e257e19e289232d40c89b5490` ==> `5ca5d191923d5c12d5fffa6d0589c1befd9d5b50`
* [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-framework-upgrade-pr-16682&var-metrics_source=All&var-chain_name=forge-1&refresh=10s&from=now-15m&to=now)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-framework-upgrade-pr-16682%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-framework-upgrade-pr-16682%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16682%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16682%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22quickRange%22%3A%20%2230m%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16682&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16682&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/15343068073)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-framework-upgrade -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-05-30T09:13:26Z
body: ### Forge is running suite `realistic_env_max_load` on `5ca5d191923d5c12d5fffa6d0589c1befd9d5b50`
* [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-e2e-pr-16682&var-metrics_source=All&var-chain_name=forge-1&refresh=10s&from=now-15m&to=now)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-e2e-pr-16682%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-e2e-pr-16682%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16682%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16682%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22quickRange%22%3A%20%2230m%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16682&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16682&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/15343068073)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-e2e -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-05-30T09:13:30Z
body: ### Forge is running suite `compat` on `ead1a2d60bb9fe8e257e19e289232d40c89b5490` ==> `5ca5d191923d5c12d5fffa6d0589c1befd9d5b50`
* [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-compat-pr-16682&var-metrics_source=All&var-chain_name=forge-0&refresh=10s&from=now-15m&to=now)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-compat-pr-16682%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-compat-pr-16682%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16682%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16682%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22quickRange%22%3A%20%2230m%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16682&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16682&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/15343068073)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-compat -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-05-30T09:24:23Z
body: ### :white_check_mark: Forge suite `compat` success on `ead1a2d60bb9fe8e257e19e289232d40c89b5490` ==> `5ca5d191923d5c12d5fffa6d0589c1befd9d5b50`
```
Compatibility test results for ead1a2d60bb9fe8e257e19e289232d40c89b5490 ==> 5ca5d191923d5c12d5fffa6d0589c1befd9d5b50 (PR)
1. Check liveness of validators at old version: ead1a2d60bb9fe8e257e19e289232d40c89b5490
compatibility::simple-validator-upgrade::liveness-check : committed: 8875.47 txn/s, latency: 3803.17 ms, (p50: 4000 ms, p70: 4100, p90: 4400 ms, p99: 4900 ms), latency samples: 296920
2. Upgrading first Validator to new version: 5ca5d191923d5c12d5fffa6d0589c1befd9d5b50
compatibility::simple-validator-upgrade::single-validator-upgrade : committed: 2598.39 txn/s, latency: 12877.89 ms, (p50: 14400 ms, p70: 14800, p90: 15200 ms, p99: 15300 ms), latency samples: 97120
3. Upgrading rest of first batch to new version: 5ca5d191923d5c12d5fffa6d0589c1befd9d5b50
compatibility::simple-validator-upgrade::half-validator-upgrade : committed: 2461.30 txn/s, latency: 13711.81 ms, (p50: 15000 ms, p70: 15700, p90: 16000 ms, p99: 16100 ms), latency samples: 92360
4. upgrading second batch to new version: 5ca5d191923d5c12d5fffa6d0589c1befd9d5b50
compatibility::simple-validator-upgrade::rest-validator-upgrade : committed: 4892.41 txn/s, latency: 6943.57 ms, (p50: 7500 ms, p70: 7800, p90: 8200 ms, p99: 8500 ms), latency samples: 175560
5. check swarm health
Compatibility test for ead1a2d60bb9fe8e257e19e289232d40c89b5490 ==> 5ca5d191923d5c12d5fffa6d0589c1befd9d5b50 passed
Test Ok
```
* [Grafana dashboard](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-compat-pr-16682&var-metrics_source=All&var-chain_name=forge-0&from=1748596417000&to=1748597060000)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-compat-pr-16682%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-compat-pr-16682%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1748596417000&end=1748597060000)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16682%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16682%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22startTime%22%3A%20%222025-05-30T09%3A13%3A37.996368%2B00%3A00%22%2C%20%22endTime%22%3A%20%222025-05-30T09%3A24%3A20.812003%2B00%3A00%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1748596417&until=1748597060&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16682&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1748596417&until=1748597060&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16682&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/15343068073)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-compat -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-05-30T09:25:15Z
body: ### :white_check_mark: Forge suite `realistic_env_max_load` success on `5ca5d191923d5c12d5fffa6d0589c1befd9d5b50`
```
two traffics test: inner traffic : committed: 13212.72 txn/s, latency: 3008.18 ms, (p50: 3000 ms, p70: 3000, p90: 3600 ms, p99: 4500 ms), latency samples: 5023960
two traffics test : committed: 99.97 txn/s, latency: 2645.76 ms, (p50: 2600 ms, p70: 3000, p90: 3900 ms, p99: 4600 ms), latency samples: 1820
Latency breakdown for phase 0: ["MempoolToBlockCreation: max: 1.644, avg: 0.660", "ConsensusProposalToOrdered: max: 0.175, avg: 0.172", "ConsensusOrderedToCommit: max: 0.318, avg: 0.275", "ConsensusProposalToCommit: max: 0.493, avg: 0.447"]
Max non-epoch-change gap was: 1 rounds at version 20001 (avg 0.00) [limit 4], 1.20s no progress at version 20001 (avg 0.12s) [limit 15].
Max epoch-change gap was: 0 rounds at version 0 (avg 0.00) [limit 4], 0.38s no progress at version 2687454 (avg 0.38s) [limit 16].
Test Ok
```
* [Grafana dashboard](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-e2e-pr-16682&var-metrics_source=All&var-chain_name=forge-1&from=1748596413000&to=1748597114000)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-e2e-pr-16682%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-e2e-pr-16682%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1748596413000&end=1748597114000)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16682%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16682%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22startTime%22%3A%20%222025-05-30T09%3A13%3A33.757124%2B00%3A00%22%2C%20%22endTime%22%3A%20%222025-05-30T09%3A25%3A14.265479%2B00%3A00%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1748596413&until=1748597114&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16682&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1748596413&until=1748597114&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16682&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/15343068073)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-e2e -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-05-30T09:26:41Z
body: ### :white_check_mark: Forge suite `framework_upgrade` success on `ead1a2d60bb9fe8e257e19e289232d40c89b5490` ==> `5ca5d191923d5c12d5fffa6d0589c1befd9d5b50`
```
Compatibility test results for ead1a2d60bb9fe8e257e19e289232d40c89b5490 ==> 5ca5d191923d5c12d5fffa6d0589c1befd9d5b50 (PR)
Upgrade the nodes to version: 5ca5d191923d5c12d5fffa6d0589c1befd9d5b50
framework_upgrade::framework-upgrade::full-framework-upgrade : committed: 1759.31 txn/s, submitted: 1763.33 txn/s, failed submission: 4.02 txn/s, expired: 4.02 txn/s, latency: 1614.42 ms, (p50: 1500 ms, p70: 1700, p90: 2100 ms, p99: 3300 ms), latency samples: 157720
framework_upgrade::framework-upgrade::full-framework-upgrade : committed: 1501.90 txn/s, submitted: 1508.74 txn/s, failed submission: 6.84 txn/s, expired: 6.84 txn/s, latency: 1704.14 ms, (p50: 1500 ms, p70: 1900, p90: 2400 ms, p99: 3700 ms), latency samples: 136200
5. check swarm health
Compatibility test for ead1a2d60bb9fe8e257e19e289232d40c89b5490 ==> 5ca5d191923d5c12d5fffa6d0589c1befd9d5b50 passed
Upgrade the remaining nodes to version: 5ca5d191923d5c12d5fffa6d0589c1befd9d5b50
framework_upgrade::framework-upgrade::full-framework-upgrade : committed: 1541.67 txn/s, submitted: 1548.53 txn/s, failed submission: 6.85 txn/s, expired: 6.85 txn/s, latency: 1753.03 ms, (p50: 1700 ms, p70: 2000, p90: 2400 ms, p99: 3400 ms), latency samples: 139500
Test Ok
```
* [Grafana dashboard](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-framework-upgrade-pr-16682&var-metrics_source=All&var-chain_name=forge-1&from=1748596411000&to=1748597200000)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-framework-upgrade-pr-16682%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-framework-upgrade-pr-16682%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1748596411000&end=1748597200000)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16682%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16682%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22startTime%22%3A%20%222025-05-30T09%3A13%3A31.572142%2B00%3A00%22%2C%20%22endTime%22%3A%20%222025-05-30T09%3A26%3A40.341066%2B00%3A00%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1748596411&until=1748597200&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16682&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1748596411&until=1748597200&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16682&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/15343068073)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-framework-upgrade -->
author_association: CONTRIBUTOR