Repository: aptos-labs/aptos-core
Pull request #16030: [10/n] Remove loaders from VM, remove legacy stores
Author: login: georgemitenkov
Created: 2025-03-01T11:04:56Z | Merged: 2025-03-07T23:57:42Z

## Description

Removes V1 loader and all legacy stores associated with it. Removes module/script caches from data store. Removes `Loader` struct because loader is simply a `ModuleStorage` trait (to be renamed?).

## How Has This Been Tested?
<!--
- Please ensure that the functionality introduced by this change is well tested and verified to work as expected.
- Ensure tests cover both happy and unhappy paths.
- List and link relevant tests.
-->

## Key Areas to Review
<!--
- Identify any critical parts of the code that require special attention or understanding. Explain why these parts are crucial to the functionality or architecture of the project.
- Point out any areas where complex logic has been implemented. Provide a brief explanation of the logic and your approach to make it easier for reviewers to follow.
- Highlight any areas where you are particularly concerned or unsure about the code's impact on the change. This can include potential performance or security issues, or compatibility with existing features.
-->

## Type of Change

- [x] Refactoring


## Which Components or Systems Does This Change Impact?

- [x] Move/Aptos Virtual Machine

## Checklist
- [x] I have read and followed the [CONTRIBUTING](https://github.com/aptos-labs/aptos-core/blob/main/CONTRIBUTING.md) doc
- [x] I have performed a self-review of my own code
- [x] I have commented my code, particularly in hard-to-understand areas
- [x] I identified and added all stakeholders and component owners affected by this change as reviewers
- [x] I tested both happy and unhappy path of the functionality
- [x] I have made corresponding changes to the documentation

<!-- Thank you for your contribution! -->



Diff:
diff --git a/Cargo.lock b/Cargo.lock
index e3e0b8ab929..b8bd8739294 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -11646,7 +11646,6 @@ dependencies = [
  "parking_lot 0.12.1",
  "proptest",
  "serde",
- "sha3 0.9.1",
  "smallbitvec",
  "triomphe",
  "typed-arena",
diff --git a/third_party/move/move-vm/runtime/Cargo.toml b/third_party/move/move-vm/runtime/Cargo.toml
index 7000cebb75e..53b2b90b5fc 100644
--- a/third_party/move/move-vm/runtime/Cargo.toml
+++ b/third_party/move/move-vm/runtime/Cargo.toml
@@ -27,7 +27,6 @@ move-vm-types = { workspace = true }
 once_cell = { workspace = true }
 parking_lot = { workspace = true }
 serde = { workspace = true }
-sha3 = { workspace = true }
 triomphe = { workspace = true }
 typed-arena = { workspace = true }
 
diff --git a/third_party/move/move-vm/runtime/src/data_cache.rs b/third_party/move/move-vm/runtime/src/data_cache.rs
index 82477f59cc7..02208cb8a86 100644
--- a/third_party/move/move-vm/runtime/src/data_cache.rs
+++ b/third_party/move/move-vm/runtime/src/data_cache.rs
@@ -3,22 +3,17 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use crate::{
-    logging::expect_no_verification_errors,
     storage::{module_storage::FunctionValueExtensionAdapter, ty_tag_converter::TypeTagConverter},
     LayoutConverter, ModuleStorage, StorageLayoutConverter,
 };
 use bytes::Bytes;
-use move_binary_format::{
-    deserializer::DeserializerConfig,
-    errors::*,
-    file_format::{CompiledModule, CompiledScript},
-};
+use move_binary_format::errors::*;
 use move_core_types::{
     account_address::AccountAddress,
     effects::{AccountChanges, ChangeSet, Changes, Op},
     gas_algebra::NumBytes,
     identifier::Identifier,
-    language_storage::{ModuleId, TypeTag},
+    language_storage::TypeTag,
     value::MoveTypeLayout,
     vm_status::StatusCode,
 };
@@ -28,11 +23,7 @@ use move_vm_types::{
     value_serde::ValueSerDeContext,
     values::{GlobalValue, Value},
 };
-use sha3::{Digest, Sha3_256};
-use std::{
-    collections::btree_map::{self, BTreeMap},
-    sync::Arc,
-};
+use std::collections::btree_map::BTreeMap;
 
 pub struct AccountDataCache {
     // The bool flag in the `data_map` indicates whether the resource contains
@@ -50,22 +41,6 @@ impl AccountDataCache {
     }
 }
 
-fn load_module_impl(
-    remote: &dyn MoveResolver,
-    account_map: &BTreeMap<AccountAddress, AccountDataCache>,
-    module_id: &ModuleId,
-) -> PartialVMResult<Bytes> {
-    if let Some(account_cache) = account_map.get(module_id.address()) {
-        if let Some((blob, _is_republishing)) = account_cache.module_map.get(module_id.name()) {
-            return Ok(blob.clone());
-        }
-    }
-    remote.get_module(module_id)?.ok_or_else(|| {
-        PartialVMError::new(StatusCode::LINKER_ERROR)
-            .with_message(format!("Linker Error: Module {} doesn't exist", module_id))
-    })
-}
-
 /// Transaction data cache. Keep updates within a transaction so they can all be published at
 /// once when the transaction succeeds.
 ///
@@ -82,27 +57,15 @@ fn load_module_impl(
 pub(crate) struct TransactionDataCache<'r> {
     remote: &'r dyn MoveResolver,
     account_map: BTreeMap<AccountAddress, AccountDataCache>,
-
-    deserializer_config: DeserializerConfig,
-
-    // Caches to help avoid duplicate deserialization calls.
-    compiled_scripts: BTreeMap<[u8; 32], Arc<CompiledScript>>,
-    compiled_modules: BTreeMap<ModuleId, (Arc<CompiledModule>, usize, [u8; 32])>,
 }
 
 impl<'r> TransactionDataCache<'r> {
     /// Create a `TransactionDataCache` with a `RemoteCache` that provides access to data
     /// not updated in the transaction.
-    pub(crate) fn new(
-        deserializer_config: DeserializerConfig,
-        remote: &'r impl MoveResolver,
-    ) -> Self {
+    pub(crate) fn new(remote: &'r impl MoveResolver) -> Self {
         TransactionDataCache {
             remote,
             account_map: BTreeMap::new(),
-            deserializer_config,
-            compiled_scripts: BTreeMap::new(),
-            compiled_modules: BTreeMap::new(),
         }
     }
 
@@ -289,73 +252,4 @@ impl<'r> TransactionDataCache<'r> {
             load_res,
         ))
     }
-
-    pub(crate) fn load_compiled_script_to_cache(
-        &mut self,
-        script_blob: &[u8],
-        hash_value: [u8; 32],
-    ) -> VMResult<Arc<CompiledScript>> {
-        let cache = &mut self.compiled_scripts;
-        match cache.entry(hash_value) {
-            btree_map::Entry::Occupied(entry) => Ok(entry.get().clone()),
-            btree_map::Entry::Vacant(entry) => {
-                let script = match CompiledScript::deserialize_with_config(
-                    script_blob,
-                    &self.deserializer_config,
-                ) {
-                    Ok(script) => script,
-                    Err(err) => {
-                        let msg = format!("[VM] deserializer for script returned error: {:?}", err);
-                        return Err(PartialVMError::new(StatusCode::CODE_DESERIALIZATION_ERROR)
-                            .with_message(msg)
-                            .finish(Location::Script));
-                    },
-                };
-                Ok(entry.insert(Arc::new(script)).clone())
-            },
-        }
-    }
-
-    pub(crate) fn load_compiled_module_to_cache(
-        &mut self,
-        id: ModuleId,
-        allow_loading_failure: bool,
-    ) -> VMResult<(Arc<CompiledModule>, usize, [u8; 32])> {
-        let cache = &mut self.compiled_modules;
-        match cache.entry(id) {
-            btree_map::Entry::Occupied(entry) => Ok(entry.get().clone()),
-            btree_map::Entry::Vacant(entry) => {
-                // bytes fetching, allow loading to fail if the flag is set
-                let bytes = match load_module_impl(self.remote, &self.account_map, entry.key())
-                    .map_err(|err| err.finish(Location::Undefined))
-                {
-                    Ok(bytes) => bytes,
-                    Err(err) if allow_loading_failure => return Err(err),
-                    Err(err) => {
-                        return Err(expect_no_verification_errors(err));
-                    },
-                };
-
-                let mut sha3_256 = Sha3_256::new();
-                sha3_256.update(&bytes);
-                let hash_value: [u8; 32] = sha3_256.finalize().into();
-
-                // for bytes obtained from the data store, they should always deserialize and verify.
-                // It is an invariant violation if they don't.
-                let module =
-                    CompiledModule::deserialize_with_config(&bytes, &self.deserializer_config)
-                        .map_err(|err| {
-                            let msg = format!("Deserialization error: {:?}", err);
-                            PartialVMError::new(StatusCode::CODE_DESERIALIZATION_ERROR)
-                                .with_message(msg)
-                                .finish(Location::Module(entry.key().clone()))
-                        })
-                        .map_err(expect_no_verification_errors)?;
-
-                Ok(entry
-                    .insert((Arc::new(module), bytes.len(), hash_value))
-                    .clone())
-            },
-        }
-    }
 }
diff --git a/third_party/move/move-vm/runtime/src/interpreter.rs b/third_party/move/move-vm/runtime/src/interpreter.rs
index 328d40fb5be..9fd5a274657 100644
--- a/third_party/move/move-vm/runtime/src/interpreter.rs
+++ b/third_party/move/move-vm/runtime/src/interpreter.rs
@@ -10,7 +10,7 @@ use crate::{
     frame_type_cache::{
         AllRuntimeCaches, FrameTypeCache, NoRuntimeCaches, PerInstructionCache, RuntimeCacheTraits,
     },
-    loader::{LazyLoadedFunction, LegacyModuleStorageAdapter, Loader, Resolver},
+    loader::{LazyLoadedFunction, Resolver},
     module_traversal::TraversalContext,
     native_extensions::NativeContextExtensions,
     native_functions::NativeContext,
@@ -116,23 +116,19 @@ impl Interpreter {
         function: LoadedFunction,
         args: Vec<Value>,
         data_store: &mut TransactionDataCache,
-        module_store: &LegacyModuleStorageAdapter,
         module_storage: &impl ModuleStorage,
         gas_meter: &mut impl GasMeter,
         traversal_context: &mut TraversalContext,
         extensions: &mut NativeContextExtensions,
-        loader: &Loader,
     ) -> VMResult<Vec<Value>> {
         InterpreterImpl::entrypoint(
             function,
             args,
             data_store,
-            module_store,
             module_storage,
             gas_meter,
             traversal_context,
             extensions,
-            loader,
         )
     }
 }
@@ -144,12 +140,10 @@ impl InterpreterImpl<'_> {
         function: LoadedFunction,
         args: Vec<Value>,
         data_store: &mut TransactionDataCache,
-        module_store: &LegacyModuleStorageAdapter,
         module_storage: &impl ModuleStorage,
         gas_meter: &mut impl GasMeter,
         traversal_context: &mut TraversalContext,
         extensions: &mut NativeContextExtensions,
-        loader: &Loader,
     ) -> VMResult<Vec<Value>> {
         let interpreter = InterpreterImpl {
             operand_stack: Stack::new(),
@@ -164,9 +158,7 @@ impl InterpreterImpl<'_> {
         // with the static RuntimeTypeCheck trait
         if interpreter.vm_config.paranoid_type_checks {
             interpreter.dispatch_execute_main::<FullRuntimeTypeCheck>(
-                loader,
                 data_store,
-                module_store,
                 module_storage,
                 gas_meter,
                 traversal_context,
@@ -176,9 +168,7 @@ impl InterpreterImpl<'_> {
             )
         } else {
             interpreter.dispatch_execute_main::<NoRuntimeTypeCheck>(
-                loader,
                 data_store,
-                module_store,
                 module_storage,
                 gas_meter,
                 traversal_context,
@@ -229,9 +219,7 @@ impl InterpreterImpl<'_> {
 
     fn dispatch_execute_main<RTTCheck: RuntimeTypeCheck>(
         self,
-        loader: &Loader,
         data_store: &mut TransactionDataCache,
-        module_store: &LegacyModuleStorageAdapter,
         module_storage: &impl ModuleStorage,
         gas_meter: &mut impl GasMeter,
         traversal_context: &mut TraversalContext,
@@ -241,9 +229,7 @@ impl InterpreterImpl<'_> {
     ) -> VMResult<Vec<Value>> {
         if self.vm_config.use_call_tree_and_instruction_cache {
             self.execute_main::<RTTCheck, AllRuntimeCaches>(
-                loader,
                 data_store,
-                module_store,
                 module_storage,
                 gas_meter,
                 traversal_context,
@@ -253,9 +239,7 @@ impl InterpreterImpl<'_> {
             )
         } else {
             self.execute_main::<RTTCheck, NoRuntimeCaches>(
-                loader,
                 data_store,
-                module_store,
                 module_storage,
                 gas_meter,
                 traversal_context,
@@ -274,9 +258,7 @@ impl InterpreterImpl<'_> {
     /// at the top of the stack (return). If the call stack is empty execution is completed.
     fn execute_main<RTTCheck: RuntimeTypeCheck, RTCaches: RuntimeCacheTraits>(
         mut self,
-        loader: &Loader,
         data_store: &mut TransactionDataCache,
-        module_store: &LegacyModuleStorageAdapter,
         module_storage: &impl ModuleStorage,
         gas_meter: &mut impl GasMeter,
         traversal_context: &mut TraversalContext,
@@ -317,7 +299,7 @@ impl InterpreterImpl<'_> {
             .map_err(|e| self.set_location(e))?;
 
         loop {
-            let resolver = current_frame.resolver(loader, module_store, module_storage);
+            let resolver = current_frame.resolver(module_storage);
             let exit_code = current_frame
                 .execute_code::<RTTCheck, RTCaches>(&resolver, &mut self, data_store, gas_meter)
                 .map_err(|err| self.attach_state_if_invariant_violation(err, &current_frame))?;
@@ -2736,14 +2718,8 @@ impl Frame {
         }
     }
 
-    fn resolver<'a>(
-        &self,
-        loader: &'a Loader,
-        module_store: &'a LegacyModuleStorageAdapter,
-        module_storage: &'a impl ModuleStorage,
-    ) -> Resolver<'a> {
-        self.function
-            .get_resolver(loader, module_store, module_storage)
+    fn resolver<'a>(&self, module_storage: &'a impl ModuleStorage) -> Resolver<'a> {
+        self.function.get_resolver(module_storage)
     }
 
     fn location(&self) -> Location {
diff --git a/third_party/move/move-vm/runtime/src/lib.rs b/third_party/move/move-vm/runtime/src/lib.rs
index 4ff9db943cc..42604a7527c 100644
--- a/third_party/move/move-vm/runtime/src/lib.rs
+++ b/third_party/move/move-vm/runtime/src/lib.rs
@@ -6,10 +6,6 @@
 #![deny(deprecated)]
 
 //! The core Move VM logic.
-//!
-//! It is a design goal for the Move VM to be independent of the Diem blockchain, so that
-//! other blockchains can use it as well. The VM isn't there yet, but hopefully will be there
-//! soon.
 
 pub mod data_cache;
 mod interpreter;
diff --git a/third_party/move/move-vm/runtime/src/loader/function.rs b/third_party/move/move-vm/runtime/src/loader/function.rs
index 96891ee8fbf..cb1eeba6ea0 100644
--- a/third_party/move/move-vm/runtime/src/loader/function.rs
+++ b/third_party/move/move-vm/runtime/src/loader/function.rs
@@ -3,10 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use crate::{
-    loader::{
-        access_specifier_loader::load_access_specifier, LegacyModuleStorageAdapter, Loader, Module,
-        Resolver, Script,
-    },
+    loader::{access_specifier_loader::load_access_specifier, Module, Resolver, Script},
     native_functions::{NativeFunction, NativeFunctions, UnboxedNativeFunction},
     storage::ty_tag_converter::TypeTagConverter,
     LayoutConverter, ModuleStorage, StorageLayoutConverter,
@@ -413,18 +410,13 @@ impl LoadedFunction {
         }
     }
 
-    pub(crate) fn get_resolver<'a>(
-        &self,
-        loader: &'a Loader,
-        module_store: &'a LegacyModuleStorageAdapter,
-        module_storage: &'a impl ModuleStorage,
-    ) -> Resolver<'a> {
+    pub(crate) fn get_resolver<'a>(&self, module_storage: &'a impl ModuleStorage) -> Resolver<'a> {
         match &self.owner {
             LoadedFunctionOwner::Module(module) => {
-                Resolver::for_module(loader, module_store, module_storage, module.clone())
+                Resolver::for_module(module_storage, module.clone())
             },
             LoadedFunctionOwner::Script(script) => {
-                Resolver::for_script(loader, module_store, module_storage, script.clone())
+                Resolver::for_script(module_storage, script.clone())
             },
         }
     }
diff --git a/third_party/move/move-vm/runtime/src/loader/mod.rs b/third_party/move/move-vm/runtime/src/loader/mod.rs
index d2efd3ffaea..b2f2927422b 100644
--- a/third_party/move/move-vm/runtime/src/loader/mod.rs
+++ b/third_party/move/move-vm/runtime/src/loader/mod.rs
@@ -3,848 +3,49 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use crate::{
-    config::VMConfig, data_cache::TransactionDataCache, logging::expect_no_verification_errors,
-    module_traversal::TraversalContext, storage::module_storage::ModuleStorage, LayoutConverter,
-    StorageLayoutConverter,
+    config::VMConfig,
+    loader::modules::{StructVariantInfo, VariantFieldInfo},
+    storage::module_storage::{FunctionValueExtensionAdapter, ModuleStorage},
+    LayoutConverter, StorageLayoutConverter,
 };
-use hashbrown::Equivalent;
-use lazy_static::lazy_static;
 use move_binary_format::{
     access::{ModuleAccess, ScriptAccess},
-    errors::{verification_error, Location, PartialVMError, PartialVMResult, VMResult},
+    errors::{PartialVMError, PartialVMResult},
     file_format::{
-        CompiledModule, CompiledScript, Constant, ConstantPoolIndex, FieldHandleIndex,
-        FieldInstantiationIndex, FunctionHandleIndex, FunctionInstantiationIndex, SignatureIndex,
-        StructDefInstantiationIndex, StructDefinitionIndex, StructFieldInformation, TableIndex,
+        Constant, ConstantPoolIndex, FieldHandleIndex, FieldInstantiationIndex,
+        FunctionHandleIndex, FunctionInstantiationIndex, SignatureIndex,
+        StructDefInstantiationIndex, StructDefinitionIndex, StructVariantHandleIndex,
+        StructVariantInstantiationIndex, VariantFieldHandleIndex, VariantFieldInstantiationIndex,
+        VariantIndex,
     },
-    IndexKind,
 };
-use move_bytecode_verifier::{self, cyclic_dependencies, dependencies};
 use move_core_types::{
-    account_address::AccountAddress,
-    gas_algebra::{NumBytes, NumTypeNodes},
+    gas_algebra::NumTypeNodes,
     identifier::IdentStr,
-    language_storage::{ModuleId, StructTag, TypeTag},
+    language_storage::{ModuleId, TypeTag},
     value::MoveTypeLayout,
     vm_status::StatusCode,
 };
-use move_vm_metrics::{Timer, VM_TIMER};
 use move_vm_types::{
     gas::GasMeter,
-    loaded_data::runtime_types::{AbilityInfo, StructType, Type},
-    sha3_256,
+    loaded_data::runtime_types::{AbilityInfo, StructType, Type, TypeBuilder},
     value_serde::FunctionValueExtension,
+    values::{AbstractFunction, SerializedFunctionData},
 };
-use parking_lot::{Mutex, RwLock};
-use std::{
-    collections::{BTreeMap, BTreeSet},
-    hash::Hash,
-    sync::Arc,
-};
+use std::sync::Arc;
 use type_loader::intern_type;
-use typed_arena::Arena;
 
 mod access_specifier_loader;
 mod function;
-mod modules;
-mod script;
-mod type_loader;
-
-use crate::{
-    loader::modules::{StructVariantInfo, VariantFieldInfo},
-    native_functions::NativeFunctions,
-    storage::{
-        loader::LoaderV2, module_storage::FunctionValueExtensionAdapter,
-        ty_tag_converter::TypeTagCache,
-    },
-};
 pub use function::{Function, LoadedFunction, LoadedFunctionOwner};
 pub(crate) use function::{
     FunctionHandle, FunctionInstantiation, LazyLoadedFunction, LazyLoadedFunctionState,
 };
+mod modules;
 pub use modules::Module;
-pub(crate) use modules::{LegacyModuleCache, LegacyModuleStorage, LegacyModuleStorageAdapter};
-use move_binary_format::file_format::{
-    StructVariantHandleIndex, StructVariantInstantiationIndex, VariantFieldHandleIndex,
-    VariantFieldInstantiationIndex, VariantIndex,
-};
-use move_vm_types::{
-    loaded_data::{runtime_types::TypeBuilder, struct_name_indexing::StructNameIndexMap},
-    values::{AbstractFunction, SerializedFunctionData},
-};
+mod script;
 pub use script::Script;
-pub(crate) use script::ScriptCache;
-
-type ScriptHash = [u8; 32];
-
-// A simple cache that offers both a HashMap and a Vector lookup.
-// Access to this cache is always under a `RwLock`.
-#[derive(Clone)]
-pub(crate) struct BinaryCache<K, V> {
-    // Notice that we are using the HashMap implementation from the hashbrown crate, not the
-    // one from std, as it allows alternative key representations to be used for lookup,
-    // making certain optimizations possible.
-    id_map: hashbrown::HashMap<K, usize>,
-    binaries: Vec<V>,
-}
-
-impl<K, V> BinaryCache<K, V>
-where
-    K: Eq + Hash,
-{
-    fn new() -> Self {
-        Self {
-            id_map: hashbrown::HashMap::new(),
-            binaries: vec![],
-        }
-    }
-
-    fn insert(&mut self, key: K, binary: V) -> &V {
-        let idx = self.binaries.len();
-        self.binaries.push(binary);
-        self.id_map.insert(key, idx);
-        self.binaries
-            .last()
-            .expect("BinaryCache: last() after push() impossible failure")
-    }
-
-    fn get<Q>(&self, key: &Q) -> Option<&V>
-    where
-        Q: Hash + Eq + Equivalent<K>,
-    {
-        let index = self.id_map.get(key)?;
-        self.binaries.get(*index)
-    }
-}
-
-// Max number of modules that can skip re-verification.
-const VERIFIED_CACHE_SIZE: usize = 100_000;
-
-// Cache for already verified modules
-lazy_static! {
-    static ref VERIFIED_MODULES: Mutex<lru::LruCache<[u8; 32], ()>> =
-        Mutex::new(lru::LruCache::new(VERIFIED_CACHE_SIZE));
-}
-
-//
-// Loader
-//
-
-pub(crate) enum Loader {
-    V1(LoaderV1),
-    V2(LoaderV2),
-}
-
-macro_rules! versioned_loader_getter {
-    ($getter:ident, $return_ty:ty) => {
-        pub(crate) fn $getter(&self) -> &$return_ty {
-            match self {
-                Self::V1(loader) => loader.$getter(),
-                Self::V2(loader) => loader.$getter(),
-            }
-        }
-    };
-}
-
-impl Loader {
-    versioned_loader_getter!(vm_config, VMConfig);
-
-    pub(crate) fn v1(natives: NativeFunctions, vm_config: VMConfig) -> Self {
-        Self::V1(LoaderV1 {
-            scripts: RwLock::new(ScriptCache::new()),
-            type_cache: TypeTagCache::empty(),
-            name_cache: StructNameIndexMap::empty(),
-            natives,
-            invalidated: RwLock::new(false),
-            module_cache_hits: RwLock::new(BTreeSet::new()),
-            vm_config,
-        })
-    }
-
-    pub(crate) fn v2(vm_config: VMConfig) -> Self {
-        Self::V2(LoaderV2::new(vm_config))
-    }
-}
-
-// A Loader is responsible to load scripts and modules and holds the cache of all loaded
-// entities. Each cache is protected by a `RwLock`. Operation in the Loader must be thread safe
-// (operating on values on the stack) and when cache needs updating the mutex must be taken.
-// The `pub(crate)` API is what a Loader offers to the runtime.
-pub(crate) struct LoaderV1 {
-    scripts: RwLock<ScriptCache>,
-    #[allow(dead_code)]
-    type_cache: TypeTagCache,
-    natives: NativeFunctions,
-    pub(crate) name_cache: StructNameIndexMap,
-
-    // The below field supports a hack to workaround well-known issues with the
-    // loader cache. This cache is not designed to support module upgrade or deletion.
-    // This leads to situations where the cache does not reflect the state of storage:
-    //
-    // 1. On module upgrade, the upgraded module is in storage, but the old one still in the cache.
-    // 2. On an abandoned code publishing transaction, the cache may contain a module which was
-    //    never committed to storage by the adapter.
-    //
-    // The solution is to add a flag to Loader marking it as 'invalidated'. For scenario (1),
-    // the VM sets the flag itself. For scenario (2), a public API allows the adapter to set
-    // the flag.
-    //
-    // If the cache is invalidated, it can (and must) still be used until there are no more
-    // sessions alive which are derived from a VM with this loader. This is because there are
-    // internal data structures derived from the loader which can become inconsistent. Therefore
-    // the adapter must explicitly call a function to flush the invalidated loader.
-    //
-    // This code (the loader) needs a complete refactoring. The new loader should
-    //   performance. This is essential for a cache like this in a multi-tenant execution
-    //   environment.
-    // - should delegate lifetime ownership to the adapter. Code loading (including verification)
-    //   is a major execution bottleneck. We should be able to reuse a cache for the lifetime of
-    //   the adapter/node, not just a VM or even session (as effectively today).
-    #[allow(dead_code)]
-    invalidated: RwLock<bool>,
-
-    // Collects the cache hits on module loads. This information can be read and reset by
-    // an adapter to reason about read/write conflicts of code publishing transactions and
-    // other transactions.
-    module_cache_hits: RwLock<BTreeSet<ModuleId>>,
-
-    vm_config: VMConfig,
-}
-
-impl LoaderV1 {
-    pub(crate) fn vm_config(&self) -> &VMConfig {
-        &self.vm_config
-    }
-
-    pub(crate) fn ty_builder(&self) -> &TypeBuilder {
-        &self.vm_config.ty_builder
-    }
-
-    //
-    // Script verification and loading
-    //
-
-    #[allow(dead_code)]
-    pub(crate) fn check_script_dependencies_and_check_gas(
-        &self,
-        module_store: &LegacyModuleStorageAdapter,
-        data_store: &mut TransactionDataCache,
-        gas_meter: &mut impl GasMeter,
-        traversal_context: &mut TraversalContext,
-        script_blob: &[u8],
-    ) -> VMResult<()> {
-        let script =
-            data_store.load_compiled_script_to_cache(script_blob, sha3_256(script_blob))?;
-        let script = traversal_context.referenced_scripts.alloc(script);
-
-        // TODO(Gas): Should we charge dependency gas for the script itself?
-        self.check_dependencies_and_charge_gas(
-            module_store,
-            data_store,
-            gas_meter,
-            &mut traversal_context.visited,
-            traversal_context.referenced_modules,
-            script.immediate_dependencies_iter(),
-        )?;
-
-        Ok(())
-    }
-
-    // Scripts are verified and dependencies are loaded.
-    // Effectively that means modules are cached from leaf to root in the dependency DAG.
-    // If a dependency error is found, loading stops and the error is returned.
-    // However all modules cached up to that point stay loaded.
-
-    // Entry point for script execution (`MoveVM::execute_script`).
-    // Verifies the script if it is not in the cache of scripts loaded.
-    // Type parameters are checked as well after every type is loaded.
-    #[allow(dead_code)]
-    pub(crate) fn load_script(
-        &self,
-        script_blob: &[u8],
-        ty_args: &[TypeTag],
-        data_store: &mut TransactionDataCache,
-        module_store: &LegacyModuleStorageAdapter,
-    ) -> VMResult<LoadedFunction> {
-        // Retrieve or load the script.
-        let hash_value = sha3_256(script_blob);
-        let mut scripts = self.scripts.write();
-        let script = match scripts.get(&hash_value) {
-            Some(cached) => cached,
-            None => {
-                let ver_script = self.deserialize_and_verify_script(
-                    script_blob,
-                    hash_value,
-                    data_store,
-                    module_store,
-                )?;
-
-                let script = Script::new(ver_script, &self.name_cache)
-                    .map_err(|e| e.finish(Location::Script))?;
-                scripts.insert(hash_value, script)
-            },
-        };
-
-        let ty_args = ty_args
-            .iter()
-            .map(|ty| self.load_type(ty, data_store, module_store))
-            .collect::<VMResult<Vec<_>>>()?;
-
-        let main = script.entry_point();
-        Type::verify_ty_arg_abilities(main.ty_param_abilities(), &ty_args).map_err(|e| {
-            e.with_message(format!(
-                "Failed to verify type arguments for script {}",
-                main.name()
-            ))
-            .finish(Location::Script)
-        })?;
-
-        Ok(LoadedFunction {
-            owner: LoadedFunctionOwner::Script(script),
-            ty_args,
-            function: main,
-        })
-    }
-
-    // The process of deserialization and verification is not and it must not be under lock.
-    // So when publishing modules through the dependency DAG it may happen that a different
-    // thread had loaded the module after this process fetched it from storage.
-    // Caching will take care of that by asking for each dependency module again under lock.
-    fn deserialize_and_verify_script(
-        &self,
-        script: &[u8],
-        hash_value: ScriptHash,
-        data_store: &mut TransactionDataCache,
-        module_store: &LegacyModuleStorageAdapter,
-    ) -> VMResult<Arc<CompiledScript>> {
-        let _timer = VM_TIMER.timer_with_label("LoaderV1::deserialize_and_verify_script");
-
-        let script = data_store.load_compiled_script_to_cache(script, hash_value)?;
-
-        // Verification:
-        //   - Local, using a bytecode verifier.
-        //   - Global, loading & verifying module dependencies.
-        move_bytecode_verifier::verify_script_with_config(
-            &self.vm_config().verifier_config,
-            script.as_ref(),
-        )?;
-        let loaded_deps = script
-            .immediate_dependencies()
-            .into_iter()
-            .map(|module_id| self.load_module(&module_id, data_store, module_store))
-            .collect::<VMResult<Vec<_>>>()?;
-        dependencies::verify_script(&script, loaded_deps.iter().map(|m| m.as_ref().as_ref()))?;
-        Ok(script)
-    }
-
-    //
-    // Module verification and loading
-    //
-
-    // Entry point for module publishing (`MoveVM::publish_module_bundle`).
-    //
-    // All modules in the bundle to be published must be loadable. This function performs all
-    // verification steps to load these modules without actually loading them into the code cache.
-    #[allow(dead_code)]
-    pub(crate) fn verify_module_bundle_for_publication(
-        &self,
-        modules: &[CompiledModule],
-        data_store: &mut TransactionDataCache,
-        module_store: &LegacyModuleStorageAdapter,
-    ) -> VMResult<()> {
-        let mut bundle_unverified: BTreeSet<_> = modules.iter().map(|m| m.self_id()).collect();
-        let mut bundle_verified = BTreeMap::new();
-        for module in modules {
-            let module_id = module.self_id();
-            bundle_unverified.remove(&module_id);
-
-            self.verify_module_for_publication(
-                module,
-                &bundle_verified,
-                &bundle_unverified,
-                data_store,
-                module_store,
-            )?;
-            bundle_verified.insert(module_id.clone(), module.clone());
-        }
-        Ok(())
-    }
-
-    // A module to be published must be loadable.
-    //
-    // This step performs all verification steps to load the module without loading it.
-    // The module is not added to the code cache. It is simply published to the data cache.
-    // See `verify_script()` for script verification steps.
-    //
-    // If a module `M` is published together with a bundle of modules (i.e., a vector of modules),
-    // - the `bundle_verified` argument tracks the modules that have already been verified in the
-    //   bundle. Basically, this represents the modules appears before `M` in the bundle vector.
-    // - the `bundle_unverified` argument tracks the modules that have not been verified when `M`
-    //   is being verified, i.e., the modules appears after `M` in the bundle vector.
-    fn verify_module_for_publication(
-        &self,
-        module: &CompiledModule,
-        bundle_verified: &BTreeMap<ModuleId, CompiledModule>,
-        bundle_unverified: &BTreeSet<ModuleId>,
-        data_store: &mut TransactionDataCache,
-        module_store: &LegacyModuleStorageAdapter,
-    ) -> VMResult<()> {
-        // Performs all verification steps to load the module without loading it, i.e., the new
-        // module will NOT show up in `module_cache`. In the module republishing case, it means
-        // that the old module is still in the `module_cache`, unless a new Loader is created,
-        // which means that a new MoveVM instance needs to be created.
-        move_bytecode_verifier::verify_module_with_config(
-            &self.vm_config().verifier_config,
-            module,
-        )?;
-        check_natives(module)?;
-
-        let mut visited = BTreeSet::new();
-        let mut friends_discovered = BTreeSet::new();
-        visited.insert(module.self_id());
-        friends_discovered.extend(module.immediate_friends());
-
-        // downward exploration of the module's dependency graph. Since we know nothing about this
-        // target module, we don't know what the module may specify as its dependencies and hence,
-        // we allow the loading of dependencies and the subsequent linking to fail.
-        self.load_and_verify_dependencies(
-            module,
-            bundle_verified,
-            data_store,
-            module_store,
-            &mut visited,
-            &mut friends_discovered,
-            /* allow_dependency_loading_failure */ true,
-        )?;
-
-        // upward exploration of the modules's dependency graph. Similar to dependency loading, as
-        // we know nothing about this target module, we don't know what the module may specify as
-        // its friends and hence, we allow the loading of friends to fail.
-        self.load_and_verify_friends(
-            friends_discovered,
-            bundle_verified,
-            bundle_unverified,
-            data_store,
-            module_store,
-            /* allow_friend_loading_failure */ true,
-        )?;
-
-        // make sure there is no cyclic dependency
-        self.verify_module_cyclic_relations(
-            module,
-            bundle_verified,
-            bundle_unverified,
-            module_store,
-        )
-    }
-
-    fn verify_module_cyclic_relations(
-        &self,
-        module: &CompiledModule,
-        bundle_verified: &BTreeMap<ModuleId, CompiledModule>,
-        bundle_unverified: &BTreeSet<ModuleId>,
-        module_store: &LegacyModuleStorageAdapter,
-    ) -> VMResult<()> {
-        // let module_cache = self.module_cache.read();
-        cyclic_dependencies::verify_module(
-            module,
-            |module_id| {
-                bundle_verified
-                    .get(module_id)
-                    .map(|module| module.immediate_dependencies())
-                    .or_else(|| {
-                        module_store
-                            .module_at(module_id)
-                            .map(|m| m.module.immediate_dependencies())
-                    })
-                    .ok_or_else(|| PartialVMError::new(StatusCode::MISSING_DEPENDENCY))
-            },
-            |module_id| {
-                if bundle_unverified.contains(module_id) {
-                    // If the module under verification declares a friend which is also in the
-                    // bundle (and positioned after this module in the bundle), we defer the cyclic
-                    // relation checking when we verify that module.
-                    Ok(vec![])
-                } else {
-                    // Otherwise, we get all the information we need to verify whether this module
-                    // creates a cyclic relation.
-                    bundle_verified
-                        .get(module_id)
-                        .map(|module| module.immediate_friends())
-                        .or_else(|| {
-                            module_store
-                                .module_at(module_id)
-                                .map(|m| m.module.immediate_friends())
-                        })
-                        .ok_or_else(|| PartialVMError::new(StatusCode::MISSING_DEPENDENCY))
-                }
-            },
-        )
-    }
-
-    //
-    // Helpers for loading and verification
-    //
-
-    pub(crate) fn load_type(
-        &self,
-        ty_tag: &TypeTag,
-        data_store: &mut TransactionDataCache,
-        module_store: &LegacyModuleStorageAdapter,
-    ) -> VMResult<Type> {
-        let resolver = |struct_tag: &StructTag| -> VMResult<Arc<StructType>> {
-            let module_id = ModuleId::new(struct_tag.address, struct_tag.module.clone());
-            self.load_module(&module_id, data_store, module_store)?;
-            module_store
-                .get_struct_type_by_identifier(&struct_tag.name, &module_id)
-                .map_err(|e| e.finish(Location::Undefined))
-        };
-        self.ty_builder().create_ty(ty_tag, resolver)
-    }
-
-    /// Traverses the whole transitive closure of dependencies, starting from the specified
-    /// modules and performs gas metering.
-    ///
-    /// The traversal follows a depth-first order, with the module itself being visited first,
-    /// followed by its dependencies, and finally its friends.
-    /// DO NOT CHANGE THE ORDER unless you have a good reason, or otherwise this could introduce
-    /// a breaking change to the gas semantics.
-    ///
-    /// This will result in the shallow-loading of the modules -- they will be read from the
-    /// storage as bytes and then deserialized, but NOT converted into the runtime representation.
-    ///
-    /// It should also be noted that this is implemented in a way that avoids the cloning of
-    /// `ModuleId`, a.k.a. heap allocations, as much as possible, which is critical for
-    /// performance.
-    ///
-    /// TODO: Revisit the order of traversal. Consider switching to alphabetical order.
-    #[allow(dead_code)]
-    pub(crate) fn check_dependencies_and_charge_gas<'a, I>(
-        &self,
-        module_store: &LegacyModuleStorageAdapter,
-        data_store: &mut TransactionDataCache,
-        gas_meter: &mut impl GasMeter,
-        visited: &mut BTreeMap<(&'a AccountAddress, &'a IdentStr), ()>,
-        referenced_modules: &'a Arena<Arc<CompiledModule>>,
-        ids: I,
-    ) -> VMResult<()>
-    where
-        I: IntoIterator<Item = (&'a AccountAddress, &'a IdentStr)>,
-        I::IntoIter: DoubleEndedIterator,
-    {
-        // Initialize the work list (stack) and the map of visited modules.
-        //
-        // TODO: Determine the reserved capacity based on the max number of dependencies allowed.
-        let mut stack = Vec::with_capacity(512);
-
-        for (addr, name) in ids.into_iter().rev() {
-            // TODO: Allow the check of special addresses to be customized.
-            if !addr.is_special() && visited.insert((addr, name), ()).is_none() {
-                stack.push((addr, name, true));
-            }
-        }
-
-        while let Some((addr, name, allow_loading_failure)) = stack.pop() {
-            // Load and deserialize the module only if it has not been cached by the loader.
-            // Otherwise, this will cause a significant regression in performance.
-            let (module, size) = match module_store.module_at_by_ref(addr, name) {
-                Some(module) => (module.module.clone(), module.size),
-                None => {
-                    let (module, size, _) = data_store.load_compiled_module_to_cache(
-                        ModuleId::new(*addr, name.to_owned()),
-                        allow_loading_failure,
-                    )?;
-                    (module, size)
-                },
-            };
-
-            // Extend the lifetime of the module to the remainder of the function body
-            // by storing it in an arena.
-            //
-            // This is needed because we need to store references derived from it in the
-            // work list.
-            let module = referenced_modules.alloc(module);
-
-            gas_meter
-                .charge_dependency(false, addr, name, NumBytes::new(size as u64))
-                .map_err(|err| {
-                    err.finish(Location::Module(ModuleId::new(*addr, name.to_owned())))
-                })?;
-
-            // Explore all dependencies and friends that have been visited yet.
-            for (addr, name) in module
-                .immediate_dependencies_iter()
-                .chain(module.immediate_friends_iter())
-                .rev()
-            {
-                // TODO: Allow the check of special addresses to be customized.
-                if !addr.is_special() && visited.insert((addr, name), ()).is_none() {
-                    stack.push((addr, name, false));
-                }
-            }
-        }
-
-        Ok(())
-    }
-
-    // The interface for module loading. Aligned with `load_type` and `load_function`, this function
-    // verifies that the module is OK instead of expect it.
-    pub(crate) fn load_module(
-        &self,
-        id: &ModuleId,
-        data_store: &mut TransactionDataCache,
-        module_store: &LegacyModuleStorageAdapter,
-    ) -> VMResult<Arc<Module>> {
-        // if the module is already in the code cache, load the cached version
-        if let Some(cached) = module_store.module_at(id) {
-            self.module_cache_hits.write().insert(id.clone());
-            return Ok(cached);
-        }
-
-        let _timer = VM_TIMER.timer_with_label("LoaderV1::load_module [cache miss]");
-
-        // otherwise, load the transitive closure of the target module
-        let module_ref = self.load_and_verify_module_and_dependencies_and_friends(
-            id,
-            &BTreeMap::new(),
-            &BTreeSet::new(),
-            data_store,
-            module_store,
-            /* allow_module_loading_failure */ true,
-        )?;
-
-        // verify that the transitive closure does not have cycles
-        self.verify_module_cyclic_relations(
-            &module_ref,
-            &BTreeMap::new(),
-            &BTreeSet::new(),
-            module_store,
-        )
-        .map_err(expect_no_verification_errors)?;
-        Ok(module_ref)
-    }
-
-    // Load, deserialize, and check the module with the bytecode verifier, without linking
-    fn load_and_verify_module(
-        &self,
-        id: &ModuleId,
-        data_store: &mut TransactionDataCache,
-        allow_loading_failure: bool,
-    ) -> VMResult<(Arc<CompiledModule>, usize)> {
-        let (module, size, hash_value) =
-            data_store.load_compiled_module_to_cache(id.clone(), allow_loading_failure)?;
-
-        if self.vm_config.paranoid_type_checks && &module.self_id() != id {
-            return Err(
-                PartialVMError::new(StatusCode::UNKNOWN_INVARIANT_VIOLATION_ERROR)
-                    .with_message("Module self id mismatch with storage".to_string())
-                    .finish(Location::Module(id.clone())),
-            );
-        }
-
-        // Verify the module if it hasn't been verified before.
-        if VERIFIED_MODULES.lock().get(&hash_value).is_none() {
-            let _timer = VM_TIMER
-                .timer_with_label("LoaderV1::load_and_verify_module [verification cache miss]");
-
-            move_bytecode_verifier::verify_module_with_config(
-                &self.vm_config.verifier_config,
-                &module,
-            )
-            .map_err(expect_no_verification_errors)?;
-
-            VERIFIED_MODULES.lock().put(hash_value, ());
-        }
-
-        check_natives(&module).map_err(expect_no_verification_errors)?;
-        Ok((module, size))
-    }
-
-    // Everything in `load_and_verify_module` and also recursively load and verify all the
-    // dependencies of the target module.
-    fn load_and_verify_module_and_dependencies(
-        &self,
-        id: &ModuleId,
-        bundle_verified: &BTreeMap<ModuleId, CompiledModule>,
-        data_store: &mut TransactionDataCache,
-        module_store: &LegacyModuleStorageAdapter,
-        visited: &mut BTreeSet<ModuleId>,
-        friends_discovered: &mut BTreeSet<ModuleId>,
-        allow_module_loading_failure: bool,
-    ) -> VMResult<Arc<Module>> {
-        // dependency loading does not permit cycles
-        if visited.contains(id) {
-            return Err(PartialVMError::new(StatusCode::CYCLIC_MODULE_DEPENDENCY)
-                .finish(Location::Undefined));
-        }
-
-        // module self-check
-        let (module, size) =
-            self.load_and_verify_module(id, data_store, allow_module_loading_failure)?;
-        visited.insert(id.clone());
-        friends_discovered.extend(module.immediate_friends());
-
-        // downward exploration of the module's dependency graph. For a module that is loaded from
-        // the data_store, we should never allow its dependencies to fail to load.
-        self.load_and_verify_dependencies(
-            &module,
-            bundle_verified,
-            data_store,
-            module_store,
-            visited,
-            friends_discovered,
-            /* allow_dependency_loading_failure */ false,
-        )?;
-
-        // if linking goes well, insert the module to the code cache
-        let module_ref =
-            module_store.insert(&self.natives, id.clone(), size, module, &self.name_cache)?;
-
-        Ok(module_ref)
-    }
-
-    // downward exploration of the module's dependency graph
-    fn load_and_verify_dependencies(
-        &self,
-        module: &CompiledModule,
-        bundle_verified: &BTreeMap<ModuleId, CompiledModule>,
-        data_store: &mut TransactionDataCache,
-        module_store: &LegacyModuleStorageAdapter,
-        visited: &mut BTreeSet<ModuleId>,
-        friends_discovered: &mut BTreeSet<ModuleId>,
-        allow_dependency_loading_failure: bool,
-    ) -> VMResult<()> {
-        // all immediate dependencies of the module being verified should be in one of the locations
-        // - the verified portion of the bundle (e.g., verified before this module)
-        // - the code cache (i.e., loaded already)
-        // - the data store (i.e., not loaded to code cache yet)
-        let mut bundle_deps = vec![];
-        let mut cached_deps = vec![];
-        for module_id in module.immediate_dependencies() {
-            if let Some(cached) = bundle_verified.get(&module_id) {
-                bundle_deps.push(cached);
-            } else {
-                let loaded = match module_store.module_at(&module_id) {
-                    None => self.load_and_verify_module_and_dependencies(
-                        &module_id,
-                        bundle_verified,
-                        data_store,
-                        module_store,
-                        visited,
-                        friends_discovered,
-                        allow_dependency_loading_failure,
-                    )?,
-                    Some(cached) => cached,
-                };
-                cached_deps.push(loaded);
-            }
-        }
-
-        // once all dependencies are loaded, do the linking check
-        let all_imm_deps = bundle_deps
-            .into_iter()
-            .chain(cached_deps.iter().map(|m| m.as_ref().as_ref()));
-        let result = dependencies::verify_module(module, all_imm_deps);
-
-        // if dependencies loading is not allowed to fail, the linking should not fail as well
-        if allow_dependency_loading_failure {
-            result
-        } else {
-            result.map_err(expect_no_verification_errors)
-        }
-    }
-
-    // Everything in `load_and_verify_module_and_dependencies` and also recursively load and verify
-    // all the friends modules of the newly loaded modules, until the friends frontier covers the
-    // whole closure.
-    fn load_and_verify_module_and_dependencies_and_friends(
-        &self,
-        id: &ModuleId,
-        bundle_verified: &BTreeMap<ModuleId, CompiledModule>,
-        bundle_unverified: &BTreeSet<ModuleId>,
-        data_store: &mut TransactionDataCache,
-        module_store: &LegacyModuleStorageAdapter,
-        allow_module_loading_failure: bool,
-    ) -> VMResult<Arc<Module>> {
-        // load the closure of the module in terms of dependency relation
-        let mut visited = BTreeSet::new();
-        let mut friends_discovered = BTreeSet::new();
-        let module_ref = self.load_and_verify_module_and_dependencies(
-            id,
-            bundle_verified,
-            data_store,
-            module_store,
-            &mut visited,
-            &mut friends_discovered,
-            allow_module_loading_failure,
-        )?;
-
-        // upward exploration of the module's friendship graph and expand the friendship frontier.
-        // For a module that is loaded from the data_store, we should never allow that its friends
-        // fail to load.
-        self.load_and_verify_friends(
-            friends_discovered,
-            bundle_verified,
-            bundle_unverified,
-            data_store,
-            module_store,
-            /* allow_friend_loading_failure */ false,
-        )?;
-        Ok(module_ref)
-    }
-
-    // upward exploration of the module's dependency graph
-    fn load_and_verify_friends(
-        &self,
-        friends_discovered: BTreeSet<ModuleId>,
-        bundle_verified: &BTreeMap<ModuleId, CompiledModule>,
-        bundle_unverified: &BTreeSet<ModuleId>,
-        data_store: &mut TransactionDataCache,
-        module_store: &LegacyModuleStorageAdapter,
-        allow_friend_loading_failure: bool,
-    ) -> VMResult<()> {
-        // for each new module discovered in the frontier, load them fully and expand the frontier.
-        // apply three filters to the new friend modules discovered
-        // - `!locked_cache.has_module(mid)`
-        //   If we friend a module that is already in the code cache, then we know that the
-        //   transitive closure of that module is loaded into the cache already, skip the loading
-        // - `!bundle_verified.contains_key(mid)`
-        //   In the case of publishing a bundle, we don't actually put the published module into
-        //   code cache. This `bundle_verified` cache is a temporary extension of the code cache
-        //   in the bundle publication scenario. If a module is already verified, we don't need to
-        //   re-load it again.
-        // - `!bundle_unverified.contains(mid)
-        //   If the module under verification declares a friend which is also in the bundle (and
-        //   positioned after this module in the bundle), we defer the loading of that module when
-        //   it is the module's turn in the bundle.
-
-        // FIXME: Is there concurrency issue?
-        let new_imm_friends: Vec<_> = friends_discovered
-            .into_iter()
-            .filter(|mid| {
-                !module_store.has_module(mid)
-                    && !bundle_verified.contains_key(mid)
-                    && !bundle_unverified.contains(mid)
-            })
-            .collect();
-
-        for module_id in new_imm_friends {
-            self.load_and_verify_module_and_dependencies_and_friends(
-                &module_id,
-                bundle_verified,
-                bundle_unverified,
-                data_store,
-                module_store,
-                allow_friend_loading_failure,
-            )?;
-        }
-        Ok(())
-    }
-}
-
-//
-// Resolver
-//
+mod type_loader;
 
 // A simple wrapper for a `Module` or a `Script` in the `Resolver`
 enum BinaryType {
@@ -856,42 +57,23 @@ enum BinaryType {
 // interpreter. It's the only API known to the interpreter and it's tailored to the interpreter
 // needs.
 pub(crate) struct Resolver<'a> {
-    #[allow(dead_code)]
-    loader: &'a Loader,
-    #[allow(dead_code)]
-    module_store: &'a LegacyModuleStorageAdapter,
     binary: BinaryType,
-
     module_storage: &'a dyn ModuleStorage,
 }
 
 impl<'a> Resolver<'a> {
-    fn for_module(
-        loader: &'a Loader,
-        module_store: &'a LegacyModuleStorageAdapter,
-        module_storage: &'a dyn ModuleStorage,
-        module: Arc<Module>,
-    ) -> Self {
+    fn for_module(module_storage: &'a dyn ModuleStorage, module: Arc<Module>) -> Self {
         let binary = BinaryType::Module(module);
         Self {
-            loader,
             binary,
-            module_store,
             module_storage,
         }
     }
 
-    fn for_script(
-        loader: &'a Loader,
-        module_store: &'a LegacyModuleStorageAdapter,
-        module_storage: &'a dyn ModuleStorage,
-        script: Arc<Script>,
-    ) -> Self {
+    fn for_script(module_storage: &'a dyn ModuleStorage, script: Arc<Script>) -> Self {
         let binary = BinaryType::Script(script);
         Self {
-            loader,
             binary,
-            module_store,
             module_storage,
         }
     }
@@ -1352,19 +534,3 @@ impl<'a> FunctionValueExtension for Resolver<'a> {
         function_value_extension.get_serialization_data(fun)
     }
 }
-
-pub(crate) fn check_natives(module: &CompiledModule) -> VMResult<()> {
-    // TODO: fix check and error code if we leave something around for native structs.
-    // For now this generates the only error test cases care about...
-    for (idx, struct_def) in module.struct_defs().iter().enumerate() {
-        if struct_def.field_information == StructFieldInformation::Native {
-            return Err(verification_error(
-                StatusCode::MISSING_DEPENDENCY,
-                IndexKind::FunctionHandle,
-                idx as TableIndex,
-            )
-            .finish(Location::Module(module.self_id())));
-        }
-    }
-    Ok(())
-}
diff --git a/third_party/move/move-vm/runtime/src/loader/modules.rs b/third_party/move/move-vm/runtime/src/loader/modules.rs
index c285f5de696..2f6130fec52 100644
--- a/third_party/move/move-vm/runtime/src/loader/modules.rs
+++ b/third_party/move/move-vm/runtime/src/loader/modules.rs
@@ -6,14 +6,13 @@ use crate::{
     loader::{
         function::{Function, FunctionHandle, FunctionInstantiation},
         type_loader::intern_type,
-        BinaryCache,
     },
     native_functions::NativeFunctions,
 };
 use move_binary_format::{
     access::ModuleAccess,
     binary_views::BinaryIndexedView,
-    errors::{Location, PartialVMError, PartialVMResult, VMResult},
+    errors::{PartialVMError, PartialVMResult},
     file_format::{
         Bytecode, CompiledModule, FieldDefinition, FieldHandleIndex, FieldInstantiationIndex,
         FunctionDefinitionIndex, SignatureIndex, StructDefinition, StructDefinitionIndex,
@@ -21,18 +20,12 @@ use move_binary_format::{
         TableIndex, VariantFieldHandleIndex, VariantFieldInstantiationIndex, VariantIndex,
     },
 };
-use move_core_types::{
-    account_address::AccountAddress,
-    identifier::{IdentStr, Identifier},
-    language_storage::ModuleId,
-    vm_status::StatusCode,
-};
+use move_core_types::{identifier::Identifier, language_storage::ModuleId, vm_status::StatusCode};
 use move_vm_metrics::{Timer, VM_TIMER};
 use move_vm_types::loaded_data::{
     runtime_types::{StructIdentifier, StructLayout, StructType, Type},
     struct_name_indexing::{StructNameIndex, StructNameIndexMap},
 };
-use parking_lot::RwLock;
 use std::{
     collections::{BTreeMap, HashMap},
     fmt::Debug,
@@ -40,147 +33,6 @@ use std::{
     sync::Arc,
 };
 
-/// This trait provides an additional api for the Session to decide where the resolved modules should be stored.
-///
-/// The default api will store the modules inside MoveVM structure but the caller can also choose to store it
-/// elsewhere as long as it implements this `ModuleStorage` trait. Doing so would allow the caller, i.e: the
-/// adapter layer, to freely decide when to drop or persist the cache as well as determining its own eviction policy.
-pub trait LegacyModuleStorage {
-    fn store_module(&self, module_id: &ModuleId, binary: Module) -> Arc<Module>;
-    fn fetch_module(&self, module_id: &ModuleId) -> Option<Arc<Module>>;
-    fn fetch_module_by_ref(&self, addr: &AccountAddress, name: &IdentStr) -> Option<Arc<Module>>;
-}
-
-pub(crate) struct LegacyModuleCache(RwLock<BinaryCache<ModuleId, Arc<Module>>>);
-
-impl LegacyModuleCache {
-    pub fn new() -> Self {
-        LegacyModuleCache(RwLock::new(BinaryCache::new()))
-    }
-
-    #[allow(dead_code)]
-    pub fn flush(&self) {
-        *self.0.write() = BinaryCache::new();
-    }
-}
-
-impl Clone for LegacyModuleCache {
-    fn clone(&self) -> Self {
-        LegacyModuleCache(RwLock::new(self.0.read().clone()))
-    }
-}
-
-impl LegacyModuleStorage for LegacyModuleCache {
-    fn store_module(&self, module_id: &ModuleId, binary: Module) -> Arc<Module> {
-        let mut cache = self.0.write();
-
-        if let Some(existing_binary) = cache.get(module_id) {
-            return existing_binary.clone();
-        }
-        cache.insert(module_id.clone(), Arc::new(binary)).clone()
-    }
-
-    fn fetch_module(&self, module_id: &ModuleId) -> Option<Arc<Module>> {
-        self.0.read().get(module_id).cloned()
-    }
-
-    fn fetch_module_by_ref(&self, addr: &AccountAddress, name: &IdentStr) -> Option<Arc<Module>> {
-        self.0.read().get(&(addr, name)).cloned()
-    }
-}
-
-// TODO(loader_v2): Remove legacy V1 loader types.
-pub(crate) struct LegacyModuleStorageAdapter {
-    modules: Arc<dyn LegacyModuleStorage>,
-}
-
-impl LegacyModuleStorageAdapter {
-    pub(crate) fn new(modules: Arc<dyn LegacyModuleStorage>) -> Self {
-        Self { modules }
-    }
-
-    // Retrieve a module by `ModuleId`. The module may have not been loaded yet in which
-    // case `None` is returned
-    pub(crate) fn module_at(&self, id: &ModuleId) -> Option<Arc<Module>> {
-        self.modules.fetch_module(id)
-    }
-
-    pub(crate) fn module_at_by_ref(
-        &self,
-        addr: &AccountAddress,
-        name: &IdentStr,
-    ) -> Option<Arc<Module>> {
-        self.modules.fetch_module_by_ref(addr, name)
-    }
-
-    pub(crate) fn insert(
-        &self,
-        natives: &NativeFunctions,
-        id: ModuleId,
-        module_size: usize,
-        compiled_module: Arc<CompiledModule>,
-        struct_name_index_map: &StructNameIndexMap,
-    ) -> VMResult<Arc<Module>> {
-        if let Some(cached) = self.module_at(&id) {
-            return Ok(cached);
-        }
-
-        let module = Module::new(natives, module_size, compiled_module, struct_name_index_map)
-            .map_err(|e| e.finish(Location::Undefined))?;
-        Ok(self.modules.store_module(&id, module))
-    }
-
-    pub(crate) fn has_module(&self, module_id: &ModuleId) -> bool {
-        self.modules.fetch_module(module_id).is_some()
-    }
-
-    // Given a ModuleId::struct_name, retrieve the `StructType` and the index associated.
-    // Return and error if the type has not been loaded
-    pub(crate) fn get_struct_type_by_identifier(
-        &self,
-        struct_name: &IdentStr,
-        module_id: &ModuleId,
-    ) -> PartialVMResult<Arc<StructType>> {
-        self.modules
-            .fetch_module(module_id)
-            .and_then(|module| {
-                let idx = module.struct_map.get(struct_name)?;
-                Some(module.structs.get(*idx)?.definition_struct_type.clone())
-            })
-            .ok_or_else(|| {
-                PartialVMError::new(StatusCode::TYPE_RESOLUTION_FAILURE).with_message(format!(
-                    "Cannot find {:?}::{:?} in cache",
-                    module_id, struct_name
-                ))
-            })
-    }
-
-    /// Given module address/name and the function name, returns the corresponding module
-    /// and function if they exist in module store cache. If not, an error is returned.
-    #[allow(dead_code)]
-    pub(crate) fn resolve_module_and_function_by_name(
-        &self,
-        module_id: &ModuleId,
-        func_name: &IdentStr,
-    ) -> PartialVMResult<(Arc<Module>, Arc<Function>)> {
-        let error = || {
-            PartialVMError::new(StatusCode::FUNCTION_RESOLUTION_FAILURE).with_message(format!(
-                "Cannot find {:?}::{:?} in cache",
-                module_id, func_name
-            ))
-        };
-
-        let module = self.modules.fetch_module(module_id).ok_or_else(error)?;
-        let function = module
-            .function_map
-            .get(func_name)
-            .and_then(|idx| module.function_defs.get(*idx))
-            .cloned()
-            .ok_or_else(error)?;
-        Ok((module, function.clone()))
-    }
-}
-
 // A Module is very similar to a binary Module but data is "transformed" to a representation
 // more appropriate to execution.
 // When code executes indexes in instructions are resolved against those runtime structure
@@ -190,6 +42,7 @@ pub struct Module {
     id: ModuleId,
 
     // size in bytes
+    #[allow(dead_code)]
     pub(crate) size: usize,
 
     // primitive pools
diff --git a/third_party/move/move-vm/runtime/src/loader/script.rs b/third_party/move/move-vm/runtime/src/loader/script.rs
index c8435bf6a7e..638b22a1897 100644
--- a/third_party/move/move-vm/runtime/src/loader/script.rs
+++ b/third_party/move/move-vm/runtime/src/loader/script.rs
@@ -1,8 +1,7 @@
 // Copyright (c) The Move Contributors
 // SPDX-License-Identifier: Apache-2.0
 
-use super::{intern_type, BinaryCache, Function, FunctionHandle, FunctionInstantiation};
-use crate::loader::ScriptHash;
+use super::{intern_type, Function, FunctionHandle, FunctionInstantiation};
 use move_binary_format::{
     access::ScriptAccess,
     binary_views::BinaryIndexedView,
@@ -202,30 +201,3 @@ impl Deref for Script {
         &self.script
     }
 }
-
-// A script cache is a map from the hash value of a script and the `Script` itself.
-// Script are added in the cache once verified and so getting a script out the cache
-// does not require further verification (except for parameters and type parameters)
-#[derive(Clone)]
-pub(crate) struct ScriptCache {
-    pub(crate) scripts: BinaryCache<ScriptHash, Arc<Script>>,
-}
-
-impl ScriptCache {
-    pub(crate) fn new() -> Self {
-        Self {
-            scripts: BinaryCache::new(),
-        }
-    }
-
-    pub(crate) fn get(&self, hash: &ScriptHash) -> Option<Arc<Script>> {
-        self.scripts.get(hash).cloned()
-    }
-
-    pub(crate) fn insert(&mut self, hash: ScriptHash, script: Script) -> Arc<Script> {
-        match self.get(&hash) {
-            Some(cached) => cached,
-            None => self.scripts.insert(hash, Arc::new(script)).clone(),
-        }
-    }
-}
diff --git a/third_party/move/move-vm/runtime/src/move_vm.rs b/third_party/move/move-vm/runtime/src/move_vm.rs
index ac45e920792..f94be12f969 100644
--- a/third_party/move/move-vm/runtime/src/move_vm.rs
+++ b/third_party/move/move-vm/runtime/src/move_vm.rs
@@ -3,9 +3,8 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use crate::{
-    data_cache::TransactionDataCache, loader::LegacyModuleStorageAdapter,
-    native_extensions::NativeContextExtensions, runtime::VMRuntime, session::Session,
-    RuntimeEnvironment,
+    data_cache::TransactionDataCache, native_extensions::NativeContextExtensions,
+    runtime::VMRuntime, session::Session, RuntimeEnvironment,
 };
 use move_vm_types::resolver::MoveResolver;
 
@@ -47,15 +46,7 @@ impl MoveVM {
     ) -> Session<'r, '_> {
         Session {
             move_vm: self,
-            data_cache: TransactionDataCache::new(
-                self.runtime
-                    .loader()
-                    .vm_config()
-                    .deserializer_config
-                    .clone(),
-                remote,
-            ),
-            module_store: LegacyModuleStorageAdapter::new(self.runtime.module_storage_v1()),
+            data_cache: TransactionDataCache::new(remote),
             native_extensions,
         }
     }
diff --git a/third_party/move/move-vm/runtime/src/runtime.rs b/third_party/move/move-vm/runtime/src/runtime.rs
index 9b470ad0486..2cf21917761 100644
--- a/third_party/move/move-vm/runtime/src/runtime.rs
+++ b/third_party/move/move-vm/runtime/src/runtime.rs
@@ -5,9 +5,7 @@
 use crate::{
     data_cache::TransactionDataCache,
     interpreter::Interpreter,
-    loader::{
-        LegacyModuleCache, LegacyModuleStorage, LegacyModuleStorageAdapter, LoadedFunction, Loader,
-    },
+    loader::LoadedFunction,
     module_traversal::TraversalContext,
     native_extensions::NativeContextExtensions,
     session::SerializedReturnValues,
@@ -26,32 +24,15 @@ use move_vm_types::{
     value_serde::ValueSerDeContext,
     values::{Locals, Reference, VMValueCast, Value},
 };
-use std::{borrow::Borrow, sync::Arc};
+use std::borrow::Borrow;
 
 /// An instantiation of the MoveVM.
-pub(crate) struct VMRuntime {
-    loader: Loader,
-    pub(crate) module_cache: Arc<LegacyModuleCache>,
-}
+pub(crate) struct VMRuntime {}
 
 impl VMRuntime {
     /// Creates a new runtime instance with provided environment.
-    pub(crate) fn new(runtime_environment: &RuntimeEnvironment) -> Self {
-        let vm_config = runtime_environment.vm_config().clone();
-        let loader = if vm_config.use_loader_v2 {
-            Loader::v2(vm_config)
-        } else {
-            let natives = runtime_environment.natives().clone();
-            Loader::v1(natives, vm_config)
-        };
-
-        VMRuntime {
-            loader,
-            // TODO(loader_v2):
-            //   We still create this cache, but if V2 loader is used, it is not used. We will
-            //   remove it in the future together with other V1 components.
-            module_cache: Arc::new(LegacyModuleCache::new()),
-        }
+    pub(crate) fn new(_runtime_environment: &RuntimeEnvironment) -> Self {
+        Self {}
     }
 
     fn deserialize_arg(
@@ -209,7 +190,6 @@ impl VMRuntime {
         function: LoadedFunction,
         serialized_args: Vec<impl Borrow<[u8]>>,
         data_store: &mut TransactionDataCache,
-        module_store: &LegacyModuleStorageAdapter,
         module_storage: &impl ModuleStorage,
         gas_meter: &mut impl GasMeter,
         traversal_context: &mut TraversalContext,
@@ -248,12 +228,10 @@ impl VMRuntime {
             function,
             deserialized_args,
             data_store,
-            module_store,
             module_storage,
             gas_meter,
             traversal_context,
             extensions,
-            &self.loader,
         )?;
         drop(timer);
 
@@ -287,7 +265,6 @@ impl VMRuntime {
         func: LoadedFunction,
         serialized_args: Vec<impl Borrow<[u8]>>,
         data_store: &mut TransactionDataCache,
-        module_store: &LegacyModuleStorageAdapter,
         gas_meter: &mut impl GasMeter,
         traversal_context: &mut TraversalContext,
         extensions: &mut NativeContextExtensions,
@@ -297,7 +274,6 @@ impl VMRuntime {
             func,
             serialized_args,
             data_store,
-            module_store,
             module_storage,
             gas_meter,
             traversal_context,
@@ -311,7 +287,6 @@ impl VMRuntime {
         ty_args: Vec<TypeTag>,
         serialized_args: Vec<impl Borrow<[u8]>>,
         data_store: &mut TransactionDataCache,
-        module_store: &LegacyModuleStorageAdapter,
         gas_meter: &mut impl GasMeter,
         traversal_context: &mut TraversalContext,
         extensions: &mut NativeContextExtensions,
@@ -324,7 +299,6 @@ impl VMRuntime {
             main,
             serialized_args,
             data_store,
-            module_store,
             code_storage,
             gas_meter,
             traversal_context,
@@ -332,12 +306,4 @@ impl VMRuntime {
         )?;
         Ok(())
     }
-
-    pub(crate) fn loader(&self) -> &Loader {
-        &self.loader
-    }
-
-    pub(crate) fn module_storage_v1(&self) -> Arc<dyn LegacyModuleStorage> {
-        self.module_cache.clone() as Arc<dyn LegacyModuleStorage>
-    }
 }
diff --git a/third_party/move/move-vm/runtime/src/session.rs b/third_party/move/move-vm/runtime/src/session.rs
index b87210ad96c..3cd59218f92 100644
--- a/third_party/move/move-vm/runtime/src/session.rs
+++ b/third_party/move/move-vm/runtime/src/session.rs
@@ -4,7 +4,7 @@
 
 use crate::{
     data_cache::TransactionDataCache,
-    loader::{LegacyModuleStorageAdapter, LoadedFunction},
+    loader::LoadedFunction,
     module_traversal::TraversalContext,
     move_vm::MoveVM,
     native_extensions::NativeContextExtensions,
@@ -32,7 +32,6 @@ use std::borrow::Borrow;
 pub struct Session<'r, 'l> {
     pub(crate) move_vm: &'l MoveVM,
     pub(crate) data_cache: TransactionDataCache<'r>,
-    pub(crate) module_store: LegacyModuleStorageAdapter,
     pub(crate) native_extensions: NativeContextExtensions<'r>,
 }
 
@@ -78,7 +77,6 @@ impl<'r, 'l> Session<'r, 'l> {
             func,
             args,
             &mut self.data_cache,
-            &self.module_store,
             gas_meter,
             traversal_context,
             &mut self.native_extensions,
@@ -104,7 +102,6 @@ impl<'r, 'l> Session<'r, 'l> {
             func,
             args,
             &mut self.data_cache,
-            &self.module_store,
             gas_meter,
             traversal_context,
             &mut self.native_extensions,
@@ -124,7 +121,6 @@ impl<'r, 'l> Session<'r, 'l> {
             func,
             args,
             &mut self.data_cache,
-            &self.module_store,
             gas_meter,
             traversal_context,
             &mut self.native_extensions,
@@ -162,7 +158,6 @@ impl<'r, 'l> Session<'r, 'l> {
             ty_args,
             args,
             &mut self.data_cache,
-            &self.module_store,
             gas_meter,
             traversal_context,
             &mut self.native_extensions,
diff --git a/third_party/move/move-vm/runtime/src/storage/environment.rs b/third_party/move/move-vm/runtime/src/storage/environment.rs
index 9a72e0d4822..473e2fc0790 100644
--- a/third_party/move/move-vm/runtime/src/storage/environment.rs
+++ b/third_party/move/move-vm/runtime/src/storage/environment.rs
@@ -3,7 +3,6 @@
 
 use crate::{
     config::VMConfig,
-    loader::check_natives,
     native_functions::{NativeFunction, NativeFunctions},
     storage::{ty_tag_converter::TypeTagCache, verified_module_cache::VERIFIED_MODULES_V2},
     Module, Script,
@@ -12,9 +11,9 @@ use ambassador::delegatable_trait;
 use bytes::Bytes;
 use move_binary_format::{
     access::{ModuleAccess, ScriptAccess},
-    errors::{Location, PartialVMError, PartialVMResult, VMResult},
-    file_format::CompiledScript,
-    CompiledModule,
+    errors::{verification_error, Location, PartialVMError, PartialVMResult, VMResult},
+    file_format::{CompiledScript, StructFieldInformation, TableIndex},
+    CompiledModule, IndexKind,
 };
 use move_bytecode_verifier::dependencies;
 use move_core_types::{
@@ -234,6 +233,7 @@ impl RuntimeEnvironment {
     }
 
     /// Returns native functions available to this runtime.
+    #[allow(dead_code)]
     pub(crate) fn natives(&self) -> &NativeFunctions {
         &self.natives
     }
@@ -327,3 +327,19 @@ impl LocallyVerifiedScript {
         self.0.immediate_dependencies_iter()
     }
 }
+
+fn check_natives(module: &CompiledModule) -> VMResult<()> {
+    // TODO: fix check and error code if we leave something around for native structs.
+    // For now this generates the only error test cases care about...
+    for (idx, struct_def) in module.struct_defs().iter().enumerate() {
+        if struct_def.field_information == StructFieldInformation::Native {
+            return Err(verification_error(
+                StatusCode::MISSING_DEPENDENCY,
+                IndexKind::FunctionHandle,
+                idx as TableIndex,
+            )
+            .finish(Location::Module(module.self_id())));
+        }
+    }
+    Ok(())
+}
diff --git a/third_party/move/move-vm/runtime/src/storage/loader.rs b/third_party/move/move-vm/runtime/src/storage/loader.rs
deleted file mode 100644
index 7bd146bdb2e..00000000000
--- a/third_party/move/move-vm/runtime/src/storage/loader.rs
+++ /dev/null
@@ -1,20 +0,0 @@
-// Copyright (c) The Move Contributors
-// SPDX-License-Identifier: Apache-2.0
-
-use crate::config::VMConfig;
-
-/// V2 implementation of loader, which is stateless - i.e., it does not contain module or script
-/// cache. Instead, module and script storages are passed to all APIs by reference.
-pub(crate) struct LoaderV2 {
-    vm_config: VMConfig,
-}
-
-impl LoaderV2 {
-    pub(crate) fn new(vm_config: VMConfig) -> Self {
-        Self { vm_config }
-    }
-
-    pub(crate) fn vm_config(&self) -> &VMConfig {
-        &self.vm_config
-    }
-}
diff --git a/third_party/move/move-vm/runtime/src/storage/mod.rs b/third_party/move/move-vm/runtime/src/storage/mod.rs
index 611bd5c0313..441c4d5283c 100644
--- a/third_party/move/move-vm/runtime/src/storage/mod.rs
+++ b/third_party/move/move-vm/runtime/src/storage/mod.rs
@@ -2,7 +2,6 @@
 // SPDX-License-Identifier: Apache-2.0
 
 pub(crate) mod depth_formula_calculator;
-pub(crate) mod loader;
 pub(crate) mod ty_tag_converter;
 mod verified_module_cache;
 


Reviews:
user: login: wrwg
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-04T06:08:28Z
user: login: gelash
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-07T19:50:45Z
user: login: gelash
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-07T19:52:47Z
user: login: georgemitenkov
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-07T19:58:08Z

Review comments:
diff_hunk: @@ -3,848 +3,49 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use crate::{
-    config::VMConfig, data_cache::TransactionDataCache, logging::expect_no_verification_errors,
-    module_traversal::TraversalContext, storage::module_storage::ModuleStorage, LayoutConverter,
-    StorageLayoutConverter,
+    config::VMConfig,
+    loader::modules::{StructVariantInfo, VariantFieldInfo},
+    storage::module_storage::{FunctionValueExtensionAdapter, ModuleStorage},
+    LayoutConverter, StorageLayoutConverter,
 };
-use hashbrown::Equivalent;
-use lazy_static::lazy_static;
 use move_binary_format::{
     access::{ModuleAccess, ScriptAccess},
-    errors::{verification_error, Location, PartialVMError, PartialVMResult, VMResult},
+    errors::{PartialVMError, PartialVMResult},
     file_format::{
-        CompiledModule, CompiledScript, Constant, ConstantPoolIndex, FieldHandleIndex,
-        FieldInstantiationIndex, FunctionHandleIndex, FunctionInstantiationIndex, SignatureIndex,
-        StructDefInstantiationIndex, StructDefinitionIndex, StructFieldInformation, TableIndex,
+        Constant, ConstantPoolIndex, FieldHandleIndex, FieldInstantiationIndex,
+        FunctionHandleIndex, FunctionInstantiationIndex, SignatureIndex,
+        StructDefInstantiationIndex, StructDefinitionIndex, StructVariantHandleIndex,
+        StructVariantInstantiationIndex, VariantFieldHandleIndex, VariantFieldInstantiationIndex,
+        VariantIndex,
     },
-    IndexKind,
 };
-use move_bytecode_verifier::{self, cyclic_dependencies, dependencies};
 use move_core_types::{
-    account_address::AccountAddress,
-    gas_algebra::{NumBytes, NumTypeNodes},
+    gas_algebra::NumTypeNodes,
     identifier::IdentStr,
-    language_storage::{ModuleId, StructTag, TypeTag},
+    language_storage::{ModuleId, TypeTag},
     value::MoveTypeLayout,
     vm_status::StatusCode,
 };
-use move_vm_metrics::{Timer, VM_TIMER};
 use move_vm_types::{
     gas::GasMeter,
-    loaded_data::runtime_types::{AbilityInfo, StructType, Type},
-    sha3_256,
+    loaded_data::runtime_types::{AbilityInfo, StructType, Type, TypeBuilder},
     value_serde::FunctionValueExtension,
+    values::{AbstractFunction, SerializedFunctionData},
 };
-use parking_lot::{Mutex, RwLock};
-use std::{
-    collections::{BTreeMap, BTreeSet},
-    hash::Hash,
-    sync::Arc,
-};
+use std::sync::Arc;
 use type_loader::intern_type;
-use typed_arena::Arena;
 
 mod access_specifier_loader;
 mod function;
-mod modules;
-mod script;
-mod type_loader;
-
-use crate::{
-    loader::modules::{StructVariantInfo, VariantFieldInfo},
-    native_functions::NativeFunctions,
-    storage::{
-        loader::LoaderV2, module_storage::FunctionValueExtensionAdapter,
-        ty_tag_converter::TypeTagCache,
-    },
-};
 pub use function::{Function, LoadedFunction, LoadedFunctionOwner};
 pub(crate) use function::{
     FunctionHandle, FunctionInstantiation, LazyLoadedFunction, LazyLoadedFunctionState,
 };
+mod modules;
 pub use modules::Module;
-pub(crate) use modules::{LegacyModuleCache, LegacyModuleStorage, LegacyModuleStorageAdapter};
-use move_binary_format::file_format::{
-    StructVariantHandleIndex, StructVariantInstantiationIndex, VariantFieldHandleIndex,
-    VariantFieldInstantiationIndex, VariantIndex,
-};
-use move_vm_types::{
-    loaded_data::{runtime_types::TypeBuilder, struct_name_indexing::StructNameIndexMap},
-    values::{AbstractFunction, SerializedFunctionData},
-};
+mod script;
 pub use script::Script;
-pub(crate) use script::ScriptCache;
-
-type ScriptHash = [u8; 32];
-
-// A simple cache that offers both a HashMap and a Vector lookup.
-// Access to this cache is always under a `RwLock`.
-#[derive(Clone)]
-pub(crate) struct BinaryCache<K, V> {
-    // Notice that we are using the HashMap implementation from the hashbrown crate, not the
-    // one from std, as it allows alternative key representations to be used for lookup,
-    // making certain optimizations possible.
-    id_map: hashbrown::HashMap<K, usize>,
-    binaries: Vec<V>,
-}
-
-impl<K, V> BinaryCache<K, V>
-where
-    K: Eq + Hash,
-{
-    fn new() -> Self {
-        Self {
-            id_map: hashbrown::HashMap::new(),
-            binaries: vec![],
-        }
-    }
-
-    fn insert(&mut self, key: K, binary: V) -> &V {
-        let idx = self.binaries.len();
-        self.binaries.push(binary);
-        self.id_map.insert(key, idx);
-        self.binaries
-            .last()
-            .expect("BinaryCache: last() after push() impossible failure")
-    }
-
-    fn get<Q>(&self, key: &Q) -> Option<&V>
-    where
-        Q: Hash + Eq + Equivalent<K>,
-    {
-        let index = self.id_map.get(key)?;
-        self.binaries.get(*index)
-    }
-}
-
-// Max number of modules that can skip re-verification.
-const VERIFIED_CACHE_SIZE: usize = 100_000;
-
-// Cache for already verified modules
-lazy_static! {
-    static ref VERIFIED_MODULES: Mutex<lru::LruCache<[u8; 32], ()>> =
-        Mutex::new(lru::LruCache::new(VERIFIED_CACHE_SIZE));
-}
-
-//
-// Loader
-//
-
-pub(crate) enum Loader {
-    V1(LoaderV1),
-    V2(LoaderV2),
-}
-
-macro_rules! versioned_loader_getter {
-    ($getter:ident, $return_ty:ty) => {
-        pub(crate) fn $getter(&self) -> &$return_ty {
-            match self {
-                Self::V1(loader) => loader.$getter(),
-                Self::V2(loader) => loader.$getter(),
-            }
-        }
-    };
-}
-
-impl Loader {
path: third_party/move/move-vm/runtime/src/loader/mod.rs
user: login: gelash
body: Proper milestone 🎉 
@zekun000 @igor-aptos @aching 
created_at: 2025-03-07T19:50:45Z
author_association: CONTRIBUTOR
diff_hunk: @@ -26,32 +24,15 @@ use move_vm_types::{
     value_serde::ValueSerDeContext,
     values::{Locals, Reference, VMValueCast, Value},
 };
-use std::{borrow::Borrow, sync::Arc};
+use std::borrow::Borrow;
 
 /// An instantiation of the MoveVM.
-pub(crate) struct VMRuntime {
-    loader: Loader,
-    pub(crate) module_cache: Arc<LegacyModuleCache>,
-}
+pub(crate) struct VMRuntime {}
path: third_party/move/move-vm/runtime/src/runtime.rs
user: login: gelash
body: this goes in a later PR?
created_at: 2025-03-07T19:51:39Z
author_association: CONTRIBUTOR
diff_hunk: @@ -26,32 +24,15 @@ use move_vm_types::{
     value_serde::ValueSerDeContext,
     values::{Locals, Reference, VMValueCast, Value},
 };
-use std::{borrow::Borrow, sync::Arc};
+use std::borrow::Borrow;
 
 /// An instantiation of the MoveVM.
-pub(crate) struct VMRuntime {
-    loader: Loader,
-    pub(crate) module_cache: Arc<LegacyModuleCache>,
-}
+pub(crate) struct VMRuntime {}
path: third_party/move/move-vm/runtime/src/runtime.rs
user: login: georgemitenkov
body: Don't have a PR for that (yet), but will make one - this can now go away (and should)
created_at: 2025-03-07T19:58:07Z
author_association: CONTRIBUTOR

PR comments:
user: login: trunk-io[bot]
created_at: 2025-03-01T11:04:59Z
body: <!-- Trunk Metrics Report -->



<details>
<summary>
⏱️ 4h 9m total CI duration on this PR
</summary>

| Slowest 15 Jobs | Cumulative Duration | Recent Runs |
|---|---|---|
| [check-dynamic-deps](https://github.com/aptos-labs/aptos-core/actions/runs/13730748932/job/38407199292) | 1h 19m | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13727974085/job/38398681084)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13730080811/job/38405232123)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13730080728/job/38405230449)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13730725242/job/38407130950)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13730748932/job/38407199292)&#8201;(+27 more) |
| [rust-cargo-deny](https://github.com/aptos-labs/aptos-core/actions/runs/13730748942/job/38407217572) | 33m | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13718817512/job/38369499650)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13727974159/job/38398721108)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13730080817/job/38405262681)&#8201;[⬜](https://github.com/aptos-labs/aptos-core/actions/runs/13730725244/job/38407147503)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13730748942/job/38407217572)&#8201;(+16 more) |
| [execution-performance / single-node-performance](https://github.com/aptos-labs/aptos-core/actions/runs/13730748921/job/38407357750) | 25m | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13730748921/job/38407357750)&#8201; |
| [rust-targeted-unit-tests](https://github.com/aptos-labs/aptos-core/actions/runs/13730748942/job/38408604457) | 22m | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13730748942/job/38408604457)&#8201; |
| [rust-targeted-unit-tests](https://github.com/aptos-labs/aptos-core/actions/runs/13704009847/job/38324948329) | 21m | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13704009847/job/38324948329)&#8201; |
| [forge-compat-test / forge](https://github.com/aptos-labs/aptos-core/actions/runs/13730748922/job/38408620964) | 15m | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13730748922/job/38408620964)&#8201; |
| [semgrep/ci](https://github.com/aptos-labs/aptos-core/actions/runs/13730748950/job/38407199326) | 11m | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13727974062/job/38398680642)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13730080737/job/38405231335)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13730080819/job/38405231980)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13730725239/job/38407130269)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13730748950/job/38407199326)&#8201;(+27 more) |
| [general-lints](https://github.com/aptos-labs/aptos-core/actions/runs/13730748942/job/38407218221) | 9m | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13718817512/job/38369498875)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13727974159/job/38398722488)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13730080817/job/38405263689)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13730725244/job/38407148162)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13730748942/job/38407218221)&#8201;(+16 more) |
| [rust-doc-tests](https://github.com/aptos-labs/aptos-core/actions/runs/13730748942/job/38407208018) | 5m | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13730748942/job/38407208018)&#8201; |
| [test-target-determinator](https://github.com/aptos-labs/aptos-core/actions/runs/13730748922/job/38407204258) | 5m | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13730748922/job/38407204258)&#8201; |
| [execution-performance / test-target-determinator](https://github.com/aptos-labs/aptos-core/actions/runs/13730748921/job/38407199811) | 5m | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13730748921/job/38407199811)&#8201; |
| [rust-lints](https://github.com/aptos-labs/aptos-core/actions/runs/13704009847/job/38324934041) | 4m | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13704009847/job/38324934041)&#8201; |
| [file_change_determinator](https://github.com/aptos-labs/aptos-core/actions/runs/13730748942/job/38407207733) | 4m | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13718817512/job/38369473746)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13727974159/job/38398694652)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13730080817/job/38405240887)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13730725244/job/38407133347)&#8201;[🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13730748942/job/38407207733)&#8201;(+17 more) |
| [rust-lints](https://github.com/aptos-labs/aptos-core/actions/runs/13710874265/job/38348035389) | 4m | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13710874265/job/38348035389)&#8201; |
| [rust-check-merge-base](https://github.com/aptos-labs/aptos-core/actions/runs/13704009847/job/38324943954) | 2m | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/13704009847/job/38324943954)&#8201; |
</details>

🚨 1 job on the last run was significantly faster/slower than expected

| Job | Duration | vs 7d avg | Delta |
|---|---|---|---|
| [execution-performance / single-node-performance](https://github.com/aptos-labs/aptos-core/actions/runs/13730748921/job/38407357750) | 25m | [21m](https://app.trunk.io/aptoslabs/ci-analytics/1_00384b63-ffe4-46c6-9b0b-1034fd44569d/1_4cfc13c8-5fda-4429-b262-c2119596dfd3?repo=aptos-labs/aptos-core&timeBased=true&numericAggregateFunctions=NUMERIC_AGGREGATE_FUNCTION_AVG&timeRange=7+days&conclusion=GIT_HUB_REPO_ACTIVITY_RUN_CONCLUSION_SUCCESS&utm_source=gh-comment) | [![+22%](https://img.shields.io/badge/+22%25-E87A30)](https://app.trunk.io/aptoslabs/ci-analytics/1_00384b63-ffe4-46c6-9b0b-1034fd44569d/1_4cfc13c8-5fda-4429-b262-c2119596dfd3?repo=aptos-labs/aptos-core&timeBased=true&numericAggregateFunctions=NUMERIC_AGGREGATE_FUNCTION_AVG&timeRange=7+days&conclusion=GIT_HUB_REPO_ACTIVITY_RUN_CONCLUSION_SUCCESS&utm_source=gh-comment) |

<sub>[settings](https://app.trunk.io/aptoslabs/settings/repo?repo=aptos-labs/aptos-core&utm_source=gh-comment#ci-analytics) ⋅ [feedback](https://form.typeform.com/to/w7Iv34u2?utm_source=gh-comment) ⋅ [docs](https://docs.trunk.io/ci-analytics/pull-request-metrics?utm_source=gh-comment) ⋅ learn more about [trunk.io](https://trunk.io?utm_source=gh-comment)</sub>
author_association: NONE
user: login: georgemitenkov
created_at: 2025-03-01T11:05:14Z
body: * **#16033** <a href="https://app.graphite.dev/github/pr/aptos-labs/aptos-core/16033?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a>
* **#16032** <a href="https://app.graphite.dev/github/pr/aptos-labs/aptos-core/16032?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a>
* **#16031** <a href="https://app.graphite.dev/github/pr/aptos-labs/aptos-core/16031?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a>
* **#16030** <a href="https://app.graphite.dev/github/pr/aptos-labs/aptos-core/16030?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a> 👈 <a href="https://app.graphite.dev/github/pr/aptos-labs/aptos-core/16030?utm_source=stack-comment-view-in-graphite" target="_blank">(View in Graphite)</a>
* **#16029** <a href="https://app.graphite.dev/github/pr/aptos-labs/aptos-core/16029?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a>
* **#16028** <a href="https://app.graphite.dev/github/pr/aptos-labs/aptos-core/16028?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a>
* **#16027** <a href="https://app.graphite.dev/github/pr/aptos-labs/aptos-core/16027?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a>
* **#16022** <a href="https://app.graphite.dev/github/pr/aptos-labs/aptos-core/16022?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a>
* **#16020** <a href="https://app.graphite.dev/github/pr/aptos-labs/aptos-core/16020?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a>
* **#16019** <a href="https://app.graphite.dev/github/pr/aptos-labs/aptos-core/16019?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a>
* **#16018** <a href="https://app.graphite.dev/github/pr/aptos-labs/aptos-core/16018?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a>
* **#15976** <a href="https://app.graphite.dev/github/pr/aptos-labs/aptos-core/15976?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a>: 1 other dependent PR ([#16005](https://github.com/aptos-labs/aptos-core/pull/16005) <a href="https://app.graphite.dev/github/pr/aptos-labs/aptos-core/16005?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a>)
* **#15974** <a href="https://app.graphite.dev/github/pr/aptos-labs/aptos-core/15974?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a>
* `main`


<h2></h2>

This stack of pull requests is managed by <a href="https://graphite.dev?utm-source=stack-comment"><b>Graphite</b></a>. Learn more about <a href="https://stacking.dev/?utm_source=stack-comment">stacking</a>.
<!-- Current dependencies on/for this PR: -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-07T23:08:09Z
body: ### Forge is running suite `realistic_env_max_load` on `366e444b6099b07ea0a2e2ea66f1d1ff6fb22aef`
* [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-e2e-pr-16030&var-metrics_source=All&var-chain_name=forge-0&refresh=10s&from=now-15m&to=now)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-e2e-pr-16030%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-e2e-pr-16030%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16030%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16030%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22quickRange%22%3A%20%2230m%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16030&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16030&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/13730748922)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-e2e -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-07T23:08:11Z
body: ### Forge is running suite `compat` on `c5eef1450476a6f7eee96d0d1953b65df04fdb91` ==> `366e444b6099b07ea0a2e2ea66f1d1ff6fb22aef`
* [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-compat-pr-16030&var-metrics_source=All&var-chain_name=forge-0&refresh=10s&from=now-15m&to=now)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-compat-pr-16030%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-compat-pr-16030%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16030%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16030%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22quickRange%22%3A%20%2230m%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16030&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16030&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/13730748922)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-compat -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-07T23:08:14Z
body: ### Forge is running suite `framework_upgrade` on `c5eef1450476a6f7eee96d0d1953b65df04fdb91` ==> `366e444b6099b07ea0a2e2ea66f1d1ff6fb22aef`
* [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-framework-upgrade-pr-16030&var-metrics_source=All&var-chain_name=forge-1&refresh=10s&from=now-15m&to=now)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-framework-upgrade-pr-16030%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-framework-upgrade-pr-16030%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16030%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16030%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22quickRange%22%3A%20%2230m%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16030&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16030&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/13730748922)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-framework-upgrade -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-07T23:18:06Z
body: ### :x: Forge suite `compat` failure on `c5eef1450476a6f7eee96d0d1953b65df04fdb91` ==> `366e444b6099b07ea0a2e2ea66f1d1ff6fb22aef`
```
Compatibility test results for c5eef1450476a6f7eee96d0d1953b65df04fdb91 ==> 366e444b6099b07ea0a2e2ea66f1d1ff6fb22aef (PR)
1. Check liveness of validators at old version: c5eef1450476a6f7eee96d0d1953b65df04fdb91
compatibility::simple-validator-upgrade::liveness-check : committed: 11999.40 txn/s, latency: 2728.19 ms, (p50: 2900 ms, p70: 3000, p90: 3100 ms, p99: 3700 ms), latency samples: 405800
2. Upgrading first Validator to new version: 366e444b6099b07ea0a2e2ea66f1d1ff6fb22aef
compatibility::simple-validator-upgrade::single-validator-upgrading : committed: 3019.58 txn/s, latency: 9649.62 ms, (p50: 10300 ms, p70: 11800, p90: 12600 ms, p99: 12700 ms), latency samples: 66500
compatibility::simple-validator-upgrade::single-validator-upgrade : committed: 3024.89 txn/s, latency: 11145.38 ms, (p50: 12100 ms, p70: 13100, p90: 13100 ms, p99: 13100 ms), latency samples: 116100
3. Upgrading rest of first batch to new version: 366e444b6099b07ea0a2e2ea66f1d1ff6fb22aef
Test Failed: Tried executing 10 txns, request counters: "success 12, failed submit [0], failed wait [8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8], by client: [(12, 0, 144): http://aptos-node-0-validator.forge-compat-pr-16030.svc:8080/v1/]"

Caused by:
    Unknown error Transaction expired, without being seen in mempool. It is guaranteed it will not be committed on chain.

Stack backtrace:
   0: anyhow::error::<impl core::convert::From<E> for anyhow::Error>::from
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/anyhow-1.0.89/src/error.rs:564:25
   1: <core::result::Result<T,F> as core::ops::try_trait::FromResidual<core::result::Result<core::convert::Infallible,E>>>::from_residual
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/result.rs:1964:27
   2: aptos_transaction_emitter_lib::emitter::transaction_executor::RestApiReliableTransactionSubmitter::submit_check_and_retry::{{closure}}
             at ./crates/transaction-emitter-lib/src/emitter/transaction_executor.rs:141:28
   3: <futures_util::future::maybe_done::MaybeDone<Fut> as core::future::future::Future>::poll
             at /usr/local/cargo/git/checkouts/futures-rs-2e928889f587a511/1845d7d/futures-util/src/future/maybe_done.rs:95:38
   4: <futures_util::future::join_all::JoinAll<F> as core::future::future::Future>::poll
             at /usr/local/cargo/git/checkouts/futures-rs-2e928889f587a511/1845d7d/futures-util/src/future/join_all.rs:143:24
   5: <aptos_transaction_emitter_lib::emitter::transaction_executor::RestApiReliableTransactionSubmitter as aptos_transaction_generator_lib::ReliableTransactionSubmitter>::execute_transactions_with_counter::{{closure}}
             at ./crates/transaction-emitter-lib/src/emitter/transaction_executor.rs:343:10
   6: <core::pin::Pin<P> as core::future::future::Future>::poll
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/future/future.rs:123:9
   7: aptos_transaction_emitter_lib::emitter::account_minter::AccountMinter::create_and_fund_seed_accounts::{{closure}}
             at ./crates/transaction-emitter-lib/src/emitter/account_minter.rs:396:18
   8: aptos_transaction_emitter_lib::emitter::account_minter::AccountMinter::create_and_fund_accounts::{{closure}}
             at ./crates/transaction-emitter-lib/src/emitter/account_minter.rs:298:10
   9: aptos_transaction_emitter_lib::emitter::account_minter::bulk_create_accounts::{{closure}}
             at ./crates/transaction-emitter-lib/src/emitter/account_minter.rs:686:14
  10: aptos_transaction_emitter_lib::emitter::TxnEmitter::start_job::{{closure}}
             at ./crates/transaction-emitter-lib/src/emitter/mod.rs:811:10
  11: aptos_transaction_emitter_lib::emitter::TxnEmitter::emit_txn_for_impl::{{closure}}
             at ./crates/transaction-emitter-lib/src/emitter/mod.rs:925:14
  12: aptos_transaction_emitter_lib::emitter::TxnEmitter::emit_txn_for::{{closure}}
             at ./crates/transaction-emitter-lib/src/emitter/mod.rs:957:14
  13: aptos_testcases::compatibility_test::stat_gather_task::{{closure}}
             at ./testsuite/testcases/src/compatibility_test.rs:53:14
  14: aptos_testcases::compatibility_test::upgrade_and_gather_stats::{{closure}}::{{closure}}
             at ./testsuite/testcases/src/compatibility_test.rs:112:14
  15: tokio::runtime::task::core::Core<T,S>::poll::{{closure}}
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/task/core.rs:331:17
  16: tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/loom/std/unsafe_cell.rs:16:9
  17: tokio::runtime::task::core::Core<T,S>::poll
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/task/core.rs:320:30
  18: tokio::runtime::task::harness::poll_future::{{closure}}
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/task/harness.rs:500:19
  19: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/panic/unwind_safe.rs:272:9
  20: std::panicking::try::do_call
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:552:40
  21: std::panicking::try
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:516:19
  22: std::panic::catch_unwind
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panic.rs:146:14
  23: tokio::runtime::task::harness::poll_future
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/task/harness.rs:488:18
  24: tokio::runtime::task::harness::Harness<T,S>::poll_inner
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/task/harness.rs:209:27
  25: tokio::runtime::task::harness::Harness<T,S>::poll
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/task/harness.rs:154:15
  26: tokio::runtime::task::raw::RawTask::poll
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/task/raw.rs:201:18
  27: tokio::runtime::task::LocalNotified<S>::run
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/task/mod.rs:436:9
  28: tokio::runtime::scheduler::multi_thread::worker::Context::run_task::{{closure}}
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/scheduler/multi_thread/worker.rs:598:18
  29: tokio::runtime::coop::with_budget
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/coop.rs:107:5
  30: tokio::runtime::coop::budget
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/coop.rs:73:5
  31: tokio::runtime::scheduler::multi_thread::worker::Context::run_task
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/scheduler/multi_thread/worker.rs:597:9
  32: tokio::runtime::scheduler::multi_thread::worker::Context::run
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/scheduler/multi_thread/worker.rs:548:24
  33: tokio::runtime::scheduler::multi_thread::worker::run::{{closure}}::{{closure}}
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/scheduler/multi_thread/worker.rs:513:21
  34: tokio::runtime::context::scoped::Scoped<T>::set
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/context/scoped.rs:40:9
  35: tokio::runtime::context::set_scheduler::{{closure}}
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/context.rs:180:26
  36: std::thread::local::LocalKey<T>::try_with
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/thread/local.rs:284:16
  37: std::thread::local::LocalKey<T>::with
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/thread/local.rs:260:9
  38: tokio::runtime::context::set_scheduler
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/context.rs:180:17
  39: tokio::runtime::scheduler::multi_thread::worker::run::{{closure}}
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/scheduler/multi_thread/worker.rs:508:9
  40: tokio::runtime::context::runtime::enter_runtime
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/context/runtime.rs:65:16
  41: tokio::runtime::scheduler::multi_thread::worker::run
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/scheduler/multi_thread/worker.rs:500:5
  42: tokio::runtime::scheduler::multi_thread::worker::Launch::launch::{{closure}}
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/scheduler/multi_thread/worker.rs:466:45
  43: <tokio::runtime::blocking::task::BlockingTask<T> as core::future::future::Future>::poll
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/blocking/task.rs:42:21
  44: tokio::runtime::task::core::Core<T,S>::poll::{{closure}}
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/task/core.rs:331:17
  45: tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/loom/std/unsafe_cell.rs:16:9
  46: tokio::runtime::task::core::Core<T,S>::poll
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/task/core.rs:320:30
  47: tokio::runtime::task::harness::poll_future::{{closure}}
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/task/harness.rs:500:19
  48: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/panic/unwind_safe.rs:272:9
  49: std::panicking::try::do_call
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:552:40
  50: std::panicking::try
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:516:19
  51: std::panic::catch_unwind
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panic.rs:146:14
  52: tokio::runtime::task::harness::poll_future
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/task/harness.rs:488:18
  53: tokio::runtime::task::harness::Harness<T,S>::poll_inner
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/task/harness.rs:209:27
  54: tokio::runtime::task::harness::Harness<T,S>::poll
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/task/harness.rs:154:15
  55: tokio::runtime::task::raw::poll
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/task/raw.rs:271:5
  56: tokio::runtime::task::raw::RawTask::poll
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/task/raw.rs:201:18
  57: tokio::runtime::task::UnownedTask<S>::run
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/task/mod.rs:473:9
  58: tokio::runtime::blocking::pool::Task::run
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/blocking/pool.rs:160:9
  59: tokio::runtime::blocking::pool::Inner::run
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/blocking/pool.rs:518:17
  60: tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}}
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/blocking/pool.rs:476:13
  61: std::sys_common::backtrace::__rust_begin_short_backtrace
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/sys_common/backtrace.rs:155:18
  62: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/thread/mod.rs:528:17
  63: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/panic/unwind_safe.rs:272:9
  64: std::panicking::try::do_call
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:552:40
  65: std::panicking::try
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:516:19
  66: std::panic::catch_unwind
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panic.rs:146:14
  67: std::thread::Builder::spawn_unchecked_::{{closure}}
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/thread/mod.rs:527:30
  68: core::ops::function::FnOnce::call_once{{vtable.shim}}
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/ops/function.rs:250:5
  69: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/alloc/src/boxed.rs:2020:9
  70: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/alloc/src/boxed.rs:2020:9
  71: std::sys::pal::unix::thread::Thread::new::thread_start
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/sys/pal/unix/thread.rs:108:17
  72: start_thread
  73: clone
Trailing Log Lines:
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/ops/function.rs:250:5
  69: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/alloc/src/boxed.rs:2020:9
  70: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/alloc/src/boxed.rs:2020:9
  71: std::sys::pal::unix::thread::Thread::new::thread_start
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/sys/pal/unix/thread.rs:108:17
  72: start_thread
  73: clone

=== BEGIN JUNIT ===
<?xml version="1.0" encoding="UTF-8"?>
<testsuites name="forge" tests="1" failures="1" errors="0" uuid="998efa5c-fe53-496e-9bbc-2ab22a749f5c">
    <testsuite name="compat" tests="1" disabled="0" errors="0" failures="1">
        <testcase name="compatibility::simple-validator-upgrade">
            <failure message="Tried executing 10 txns, request counters: &quot;success 12, failed submit [0], failed wait [8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8], by client: [(12, 0, 144): http://aptos-node-0-validator.forge-compat-pr-16030.svc:8080/v1/]&quot;

Caused by:
    Unknown error Transaction expired, without being seen in mempool. It is guaranteed it will not be committed on chain.

Stack backtrace:
   0: anyhow::error::&lt;impl core::convert::From&lt;E&gt; for anyhow::Error&gt;::from
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/anyhow-1.0.89/src/error.rs:564:25
   1: &lt;core::result::Result&lt;T,F&gt; as core::ops::try_trait::FromResidual&lt;core::result::Result&lt;core::convert::Infallible,E&gt;&gt;&gt;::from_residual
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/result.rs:1964:27
   2: aptos_transaction_emitter_lib::emitter::transaction_executor::RestApiReliableTransactionSubmitter::submit_check_and_retry::{{closure}}
             at ./crates/transaction-emitter-lib/src/emitter/transaction_executor.rs:141:28
   3: &lt;futures_util::future::maybe_done::MaybeDone&lt;Fut&gt; as core::future::future::Future&gt;::poll
             at /usr/local/cargo/git/checkouts/futures-rs-2e928889f587a511/1845d7d/futures-util/src/future/maybe_done.rs:95:38
   4: &lt;futures_util::future::join_all::JoinAll&lt;F&gt; as core::future::future::Future&gt;::poll
             at /usr/local/cargo/git/checkouts/futures-rs-2e928889f587a511/1845d7d/futures-util/src/future/join_all.rs:143:24
   5: &lt;aptos_transaction_emitter_lib::emitter::transaction_executor::RestApiReliableTransactionSubmitter as aptos_transaction_generator_lib::ReliableTransactionSubmitter&gt;::execute_transactions_with_counter::{{closure}}
             at ./crates/transaction-emitter-lib/src/emitter/transaction_executor.rs:343:10
   6: &lt;core::pin::Pin&lt;P&gt; as core::future::future::Future&gt;::poll
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/future/future.rs:123:9
   7: aptos_transaction_emitter_lib::emitter::account_minter::AccountMinter::create_and_fund_seed_accounts::{{closure}}
             at ./crates/transaction-emitter-lib/src/emitter/account_minter.rs:396:18
   8: aptos_transaction_emitter_lib::emitter::account_minter::AccountMinter::create_and_fund_accounts::{{closure}}
             at ./crates/transaction-emitter-lib/src/emitter/account_minter.rs:298:10
   9: aptos_transaction_emitter_lib::emitter::account_minter::bulk_create_accounts::{{closure}}
             at ./crates/transaction-emitter-lib/src/emitter/account_minter.rs:686:14
  10: aptos_transaction_emitter_lib::emitter::TxnEmitter::start_job::{{closure}}
             at ./crates/transaction-emitter-lib/src/emitter/mod.rs:811:10
  11: aptos_transaction_emitter_lib::emitter::TxnEmitter::emit_txn_for_impl::{{closure}}
             at ./crates/transaction-emitter-lib/src/emitter/mod.rs:925:14
  12: aptos_transaction_emitter_lib::emitter::TxnEmitter::emit_txn_for::{{closure}}
             at ./crates/transaction-emitter-lib/src/emitter/mod.rs:957:14
  13: aptos_testcases::compatibility_test::stat_gather_task::{{closure}}
             at ./testsuite/testcases/src/compatibility_test.rs:53:14
  14: aptos_testcases::compatibility_test::upgrade_and_gather_stats::{{closure}}::{{closure}}
             at ./testsuite/testcases/src/compatibility_test.rs:112:14
  15: tokio::runtime::task::core::Core&lt;T,S&gt;::poll::{{closure}}
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/task/core.rs:331:17
  16: tokio::loom::std::unsafe_cell::UnsafeCell&lt;T&gt;::with_mut
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/loom/std/unsafe_cell.rs:16:9
  17: tokio::runtime::task::core::Core&lt;T,S&gt;::poll
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/task/core.rs:320:30
  18: tokio::runtime::task::harness::poll_future::{{closure}}
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/task/harness.rs:500:19
  19: &lt;core::panic::unwind_safe::AssertUnwindSafe&lt;F&gt; as core::ops::function::FnOnce&lt;()&gt;&gt;::call_once
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/panic/unwind_safe.rs:272:9
  20: std::panicking::try::do_call
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:552:40
  21: std::panicking::try
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:516:19
  22: std::panic::catch_unwind
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panic.rs:146:14
  23: tokio::runtime::task::harness::poll_future
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/task/harness.rs:488:18
  24: tokio::runtime::task::harness::Harness&lt;T,S&gt;::poll_inner
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/task/harness.rs:209:27
  25: tokio::runtime::task::harness::Harness&lt;T,S&gt;::poll
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/task/harness.rs:154:15
  26: tokio::runtime::task::raw::RawTask::poll
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/task/raw.rs:201:18
  27: tokio::runtime::task::LocalNotified&lt;S&gt;::run
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/task/mod.rs:436:9
  28: tokio::runtime::scheduler::multi_thread::worker::Context::run_task::{{closure}}
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/scheduler/multi_thread/worker.rs:598:18
  29: tokio::runtime::coop::with_budget
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/coop.rs:107:5
  30: tokio::runtime::coop::budget
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/coop.rs:73:5
  31: tokio::runtime::scheduler::multi_thread::worker::Context::run_task
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/scheduler/multi_thread/worker.rs:597:9
  32: tokio::runtime::scheduler::multi_thread::worker::Context::run
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/scheduler/multi_thread/worker.rs:548:24
  33: tokio::runtime::scheduler::multi_thread::worker::run::{{closure}}::{{closure}}
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/scheduler/multi_thread/worker.rs:513:21
  34: tokio::runtime::context::scoped::Scoped&lt;T&gt;::set
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/context/scoped.rs:40:9
  35: tokio::runtime::context::set_scheduler::{{closure}}
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/context.rs:180:26
  36: std::thread::local::LocalKey&lt;T&gt;::try_with
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/thread/local.rs:284:16
  37: std::thread::local::LocalKey&lt;T&gt;::with
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/thread/local.rs:260:9
  38: tokio::runtime::context::set_scheduler
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/context.rs:180:17
  39: tokio::runtime::scheduler::multi_thread::worker::run::{{closure}}
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/scheduler/multi_thread/worker.rs:508:9
  40: tokio::runtime::context::runtime::enter_runtime
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/context/runtime.rs:65:16
  41: tokio::runtime::scheduler::multi_thread::worker::run
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/scheduler/multi_thread/worker.rs:500:5
  42: tokio::runtime::scheduler::multi_thread::worker::Launch::launch::{{closure}}
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/scheduler/multi_thread/worker.rs:466:45
  43: &lt;tokio::runtime::blocking::task::BlockingTask&lt;T&gt; as core::future::future::Future&gt;::poll
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/blocking/task.rs:42:21
  44: tokio::runtime::task::core::Core&lt;T,S&gt;::poll::{{closure}}
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/task/core.rs:331:17
  45: tokio::loom::std::unsafe_cell::UnsafeCell&lt;T&gt;::with_mut
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/loom/std/unsafe_cell.rs:16:9
  46: tokio::runtime::task::core::Core&lt;T,S&gt;::poll
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/task/core.rs:320:30
  47: tokio::runtime::task::harness::poll_future::{{closure}}
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/task/harness.rs:500:19
  48: &lt;core::panic::unwind_safe::AssertUnwindSafe&lt;F&gt; as core::ops::function::FnOnce&lt;()&gt;&gt;::call_once
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/panic/unwind_safe.rs:272:9
  49: std::panicking::try::do_call
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:552:40
  50: std::panicking::try
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:516:19
  51: std::panic::catch_unwind
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panic.rs:146:14
  52: tokio::runtime::task::harness::poll_future
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/task/harness.rs:488:18
  53: tokio::runtime::task::harness::Harness&lt;T,S&gt;::poll_inner
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/task/harness.rs:209:27
  54: tokio::runtime::task::harness::Harness&lt;T,S&gt;::poll
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/task/harness.rs:154:15
  55: tokio::runtime::task::raw::poll
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/task/raw.rs:271:5
  56: tokio::runtime::task::raw::RawTask::poll
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/task/raw.rs:201:18
  57: tokio::runtime::task::UnownedTask&lt;S&gt;::run
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/task/mod.rs:473:9
  58: tokio::runtime::blocking::pool::Task::run
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/blocking/pool.rs:160:9
  59: tokio::runtime::blocking::pool::Inner::run
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/blocking/pool.rs:518:17
  60: tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}}
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/blocking/pool.rs:476:13
  61: std::sys_common::backtrace::__rust_begin_short_backtrace
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/sys_common/backtrace.rs:155:18
  62: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/thread/mod.rs:528:17
  63: &lt;core::panic::unwind_safe::AssertUnwindSafe&lt;F&gt; as core::ops::function::FnOnce&lt;()&gt;&gt;::call_once
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/panic/unwind_safe.rs:272:9
  64: std::panicking::try::do_call
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:552:40
  65: std::panicking::try
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:516:19
  66: std::panic::catch_unwind
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panic.rs:146:14
  67: std::thread::Builder::spawn_unchecked_::{{closure}}
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/thread/mod.rs:527:30
  68: core::ops::function::FnOnce::call_once{{vtable.shim}}
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/ops/function.rs:250:5
  69: &lt;alloc::boxed::Box&lt;F,A&gt; as core::ops::function::FnOnce&lt;Args&gt;&gt;::call_once
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/alloc/src/boxed.rs:2020:9
  70: &lt;alloc::boxed::Box&lt;F,A&gt; as core::ops::function::FnOnce&lt;Args&gt;&gt;::call_once
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/alloc/src/boxed.rs:2020:9
  71: std::sys::pal::unix::thread::Thread::new::thread_start
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/sys/pal/unix/thread.rs:108:17
  72: start_thread
  73: clone"/>
        </testcase>
    </testsuite>
</testsuites>
=== END JUNIT ===

Swarm logs can be found here: See fgi output for more information.
{"level":"INFO","source":{"package":"aptos_forge","file":"testsuite/forge/src/backend/k8s/cluster_helper.rs:381"},"thread_name":"main","hostname":"forge-compat-pr-16030-1741388896-c5eef1450476a6f7eee96d0d1953b6","timestamp":"2025-03-07T23:18:01.959709Z","message":"Deleting namespace forge-compat-pr-16030: Some(NamespaceStatus { conditions: None, phase: Some(\"Terminating\") })"}
{"level":"INFO","source":{"package":"aptos_forge","file":"testsuite/forge/src/backend/k8s/cluster_helper.rs:398"},"thread_name":"main","hostname":"forge-compat-pr-16030-1741388896-c5eef1450476a6f7eee96d0d1953b6","timestamp":"2025-03-07T23:18:01.959733Z","message":"aptos-node resources for Forge removed in namespace: forge-compat-pr-16030"}

failures:
    compatibility::simple-validator-upgrade

Failed to run tests:
Tests Failed
test result: FAILED. 0 passed; 1 failed; 0 filtered out

Error: Tests Failed

Stack backtrace:
   0: anyhow::error::<impl anyhow::Error>::msg
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/anyhow-1.0.89/src/error.rs:85:36
   1: aptos_forge::runner::Forge<F>::run
             at ./testsuite/forge/src/runner.rs:358:13
   2: forge::run_forge_with_changelog
             at ./testsuite/forge-cli/src/main.rs:428:24
   3: forge::main
             at ./testsuite/forge-cli/src/main.rs:331:21
   4: core::ops::function::FnOnce::call_once
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/ops/function.rs:250:5
   5: std::sys_common::backtrace::__rust_begin_short_backtrace
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/sys_common/backtrace.rs:155:18
   6: std::rt::lang_start::{{closure}}
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/rt.rs:166:18
   7: core::ops::function::impls::<impl core::ops::function::FnOnce<A> for &F>::call_once
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/ops/function.rs:284:13
   8: std::panicking::try::do_call
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:552:40
   9: std::panicking::try
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:516:19
  10: std::panic::catch_unwind
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panic.rs:146:14
  11: std::rt::lang_start_internal::{{closure}}
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/rt.rs:148:48
  12: std::panicking::try::do_call
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:552:40
  13: std::panicking::try
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:516:19
  14: std::panic::catch_unwind
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panic.rs:146:14
  15: std::rt::lang_start_internal
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/rt.rs:148:20
  16: main
  17: __libc_start_main
  18: _start
Debugging output:
NAME                                    READY   STATUS      RESTARTS   AGE
aptos-node-0-validator-0                1/1     Running     0          3m51s
aptos-node-1-validator-0                1/1     Running     0          5m53s
aptos-node-2-validator-0                1/1     Running     0          8m41s
aptos-node-3-validator-0                1/1     Running     0          8m41s
forge-testnet-deployer-g8k2b            0/1     Completed   0          9m30s
genesis-aptos-genesis-eforge117-jjlzs   0/1     Completed   0          8m42s

```
* [Grafana dashboard](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-compat-pr-16030&var-metrics_source=All&var-chain_name=forge-0&from=1741388897000&to=1741389484000)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-compat-pr-16030%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-compat-pr-16030%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1741388897000&end=1741389484000)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16030%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16030%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22startTime%22%3A%20%222025-03-07T23%3A08%3A17.559457%2B00%3A00%22%2C%20%22endTime%22%3A%20%222025-03-07T23%3A18%3A04.325837%2B00%3A00%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1741388897&until=1741389484&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16030&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1741388897&until=1741389484&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16030&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/13730748922)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-compat -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-07T23:20:58Z
body: ### :white_check_mark: Forge suite `realistic_env_max_load` success on `366e444b6099b07ea0a2e2ea66f1d1ff6fb22aef`
```
two traffics test: inner traffic : committed: 12237.75 txn/s, latency: 3252.64 ms, (p50: 2700 ms, p70: 3300, p90: 5100 ms, p99: 10200 ms), latency samples: 4653060
two traffics test : committed: 99.96 txn/s, latency: 3955.75 ms, (p50: 2600 ms, p70: 4400, p90: 8600 ms, p99: 10200 ms), latency samples: 1760
Latency breakdown for phase 0: ["MempoolToBlockCreation: max: 1.250, avg: 0.569", "ConsensusProposalToOrdered: max: 0.324, avg: 0.305", "ConsensusOrderedToCommit: max: 0.586, avg: 0.448", "ConsensusProposalToCommit: max: 0.893, avg: 0.753"]
Max non-epoch-change gap was: 1 rounds at version 45369 (avg 0.00) [limit 4], 1.94s no progress at version 45369 (avg 0.21s) [limit 15].
Max epoch-change gap was: 0 rounds at version 0 (avg 0.00) [limit 4], 0.98s no progress at version 2031235 (avg 0.98s) [limit 16].
Test Ok
```
* [Grafana dashboard](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-e2e-pr-16030&var-metrics_source=All&var-chain_name=forge-0&from=1741388894000&to=1741389656000)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-e2e-pr-16030%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-e2e-pr-16030%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1741388894000&end=1741389656000)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16030%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16030%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22startTime%22%3A%20%222025-03-07T23%3A08%3A14.938953%2B00%3A00%22%2C%20%22endTime%22%3A%20%222025-03-07T23%3A20%3A56.520815%2B00%3A00%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1741388894&until=1741389656&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16030&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1741388894&until=1741389656&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16030&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/13730748922)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-e2e -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-07T23:21:34Z
body: ### :white_check_mark: Forge suite `framework_upgrade` success on `c5eef1450476a6f7eee96d0d1953b65df04fdb91` ==> `366e444b6099b07ea0a2e2ea66f1d1ff6fb22aef`
```
Compatibility test results for c5eef1450476a6f7eee96d0d1953b65df04fdb91 ==> 366e444b6099b07ea0a2e2ea66f1d1ff6fb22aef (PR)
Upgrade the nodes to version: 366e444b6099b07ea0a2e2ea66f1d1ff6fb22aef
framework_upgrade::framework-upgrade::full-framework-upgrade : committed: 1666.08 txn/s, submitted: 1673.48 txn/s, failed submission: 7.04 txn/s, expired: 7.40 txn/s, latency: 1732.03 ms, (p50: 1500 ms, p70: 1800, p90: 2300 ms, p99: 3400 ms), latency samples: 151507
framework_upgrade::framework-upgrade::full-framework-upgrade : committed: 1707.44 txn/s, submitted: 1712.27 txn/s, failed submission: 4.83 txn/s, expired: 4.83 txn/s, latency: 1712.63 ms, (p50: 1600 ms, p70: 1800, p90: 2200 ms, p99: 3100 ms), latency samples: 155581
5. check swarm health
Compatibility test for c5eef1450476a6f7eee96d0d1953b65df04fdb91 ==> 366e444b6099b07ea0a2e2ea66f1d1ff6fb22aef passed
Upgrade the remaining nodes to version: 366e444b6099b07ea0a2e2ea66f1d1ff6fb22aef
framework_upgrade::framework-upgrade::full-framework-upgrade : committed: 1607.78 txn/s, submitted: 1612.69 txn/s, failed submission: 4.91 txn/s, expired: 4.91 txn/s, latency: 1836.22 ms, (p50: 1800 ms, p70: 2100, p90: 2500 ms, p99: 3000 ms), latency samples: 144080
Test Ok
```
* [Grafana dashboard](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-framework-upgrade-pr-16030&var-metrics_source=All&var-chain_name=forge-1&from=1741388902000&to=1741389692000)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-framework-upgrade-pr-16030%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-framework-upgrade-pr-16030%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1741388902000&end=1741389692000)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16030%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16030%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22startTime%22%3A%20%222025-03-07T23%3A08%3A22.772836%2B00%3A00%22%2C%20%22endTime%22%3A%20%222025-03-07T23%3A21%3A32.878399%2B00%3A00%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1741388902&until=1741389692&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16030&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1741388902&until=1741389692&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16030&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/13730748922)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-framework-upgrade -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-07T23:37:13Z
body: ### Forge is running suite `compat` on `c5eef1450476a6f7eee96d0d1953b65df04fdb91` ==> `366e444b6099b07ea0a2e2ea66f1d1ff6fb22aef`
* [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-compat-pr-16030&var-metrics_source=All&var-chain_name=forge-0&refresh=10s&from=now-15m&to=now)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-compat-pr-16030%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-compat-pr-16030%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16030%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16030%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22quickRange%22%3A%20%2230m%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16030&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16030&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/13730748922)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-compat -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-03-07T23:51:20Z
body: ### :white_check_mark: Forge suite `compat` success on `c5eef1450476a6f7eee96d0d1953b65df04fdb91` ==> `366e444b6099b07ea0a2e2ea66f1d1ff6fb22aef`
```
Compatibility test results for c5eef1450476a6f7eee96d0d1953b65df04fdb91 ==> 366e444b6099b07ea0a2e2ea66f1d1ff6fb22aef (PR)
1. Check liveness of validators at old version: c5eef1450476a6f7eee96d0d1953b65df04fdb91
compatibility::simple-validator-upgrade::liveness-check : committed: 11728.13 txn/s, latency: 2805.84 ms, (p50: 2900 ms, p70: 3100, p90: 3300 ms, p99: 4700 ms), latency samples: 388080
2. Upgrading first Validator to new version: 366e444b6099b07ea0a2e2ea66f1d1ff6fb22aef
compatibility::simple-validator-upgrade::single-validator-upgrading : committed: 3053.55 txn/s, latency: 9525.51 ms, (p50: 10200 ms, p70: 11500, p90: 12300 ms, p99: 12400 ms), latency samples: 68300
compatibility::simple-validator-upgrade::single-validator-upgrade : committed: 3084.07 txn/s, latency: 11009.56 ms, (p50: 11900 ms, p70: 12800, p90: 12800 ms, p99: 12900 ms), latency samples: 120600
3. Upgrading rest of first batch to new version: 366e444b6099b07ea0a2e2ea66f1d1ff6fb22aef
compatibility::simple-validator-upgrade::half-validator-upgrading : committed: 2998.29 txn/s, latency: 9774.26 ms, (p50: 10300 ms, p70: 11700, p90: 12400 ms, p99: 12500 ms), latency samples: 67080
compatibility::simple-validator-upgrade::half-validator-upgrade : committed: 3006.72 txn/s, latency: 11266.29 ms, (p50: 12100 ms, p70: 12700, p90: 12800 ms, p99: 12800 ms), latency samples: 118680
4. upgrading second batch to new version: 366e444b6099b07ea0a2e2ea66f1d1ff6fb22aef
compatibility::simple-validator-upgrade::rest-validator-upgrading : committed: 1759.07 txn/s, submitted: 1759.25 txn/s, expired: 0.18 txn/s, latency: 5431.55 ms, (p50: 6200 ms, p70: 6600, p90: 6900 ms, p99: 6900 ms), latency samples: 107509
compatibility::simple-validator-upgrade::rest-validator-upgrade : committed: 2145.38 txn/s, submitted: 2145.51 txn/s, expired: 0.12 txn/s, latency: 6158.09 ms, (p50: 6800 ms, p70: 7000, p90: 7200 ms, p99: 7300 ms), latency samples: 190569
5. check swarm health
Compatibility test for c5eef1450476a6f7eee96d0d1953b65df04fdb91 ==> 366e444b6099b07ea0a2e2ea66f1d1ff6fb22aef passed
Test Ok
```
* [Grafana dashboard](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-compat-pr-16030&var-metrics_source=All&var-chain_name=forge-0&from=1741390641000&to=1741391478000)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-compat-pr-16030%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-compat-pr-16030%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1741390641000&end=1741391478000)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16030%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16030%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22startTime%22%3A%20%222025-03-07T23%3A37%3A21.981259%2B00%3A00%22%2C%20%22endTime%22%3A%20%222025-03-07T23%3A51%3A18.236596%2B00%3A00%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1741390641&until=1741391478&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16030&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1741390641&until=1741391478&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16030&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/13730748922)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-compat -->
author_association: CONTRIBUTOR