Repository: sigp/lighthouse
Pull request #7059: Manual finalization endpoint
Author: login: eserilev
Labels: name: v7.0.0-beta.2
description: Second hacky Holesky rescue release
Created: 2025-02-28T19:45:15Z | Merged: 2025-03-03T05:31:52Z

Introduce a new lighthouse endpoint that allows a user to force a manual finalization. This migrates data to the freezer db and prunes sidechains which may help reduce disk space issues on holesky at the moment.



Diff:
diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs
index f7f186704f5..fc4b4fc7a06 100644
--- a/beacon_node/beacon_chain/src/beacon_chain.rs
+++ b/beacon_node/beacon_chain/src/beacon_chain.rs
@@ -42,7 +42,7 @@ use crate::light_client_optimistic_update_verification::{
     Error as LightClientOptimisticUpdateError, VerifiedLightClientOptimisticUpdate,
 };
 use crate::light_client_server_cache::LightClientServerCache;
-use crate::migrate::BackgroundMigrator;
+use crate::migrate::{BackgroundMigrator, ManualFinalizationNotification};
 use crate::naive_aggregation_pool::{
     AggregatedAttestationMap, Error as NaiveAggregationError, NaiveAggregationPool,
     SyncContributionAggregateMap,
@@ -118,8 +118,8 @@ use std::sync::Arc;
 use std::time::Duration;
 use store::iter::{BlockRootsIterator, ParentRootBlockIterator, StateRootsIterator};
 use store::{
-    BlobSidecarListFromRoot, DatabaseBlock, Error as DBError, HotColdDB, KeyValueStore,
-    KeyValueStoreOp, StoreItem, StoreOp,
+    BlobSidecarListFromRoot, DatabaseBlock, Error as DBError, HotColdDB, HotStateSummary,
+    KeyValueStore, KeyValueStoreOp, StoreItem, StoreOp,
 };
 use task_executor::{ShutdownReason, TaskExecutor};
 use tokio::sync::oneshot;
@@ -1707,6 +1707,41 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         }
     }
 
+    pub fn manually_finalize_state(
+        &self,
+        state_root: Hash256,
+        checkpoint: Checkpoint,
+    ) -> Result<(), Error> {
+        let HotStateSummary {
+            slot,
+            latest_block_root,
+            ..
+        } = self
+            .store
+            .load_hot_state_summary(&state_root)
+            .map_err(BeaconChainError::DBError)?
+            .ok_or(BeaconChainError::MissingHotStateSummary(state_root))?;
+
+        if slot != checkpoint.epoch.start_slot(T::EthSpec::slots_per_epoch())
+            || latest_block_root != *checkpoint.root
+        {
+            return Err(BeaconChainError::InvalidCheckpoint {
+                state_root,
+                checkpoint,
+            });
+        }
+
+        let notif = ManualFinalizationNotification {
+            state_root: state_root.into(),
+            checkpoint,
+            head_tracker: self.head_tracker.clone(),
+            genesis_block_root: self.genesis_block_root,
+        };
+
+        self.store_migrator.process_manual_finalization(notif);
+        Ok(())
+    }
+
     /// Returns an aggregated `Attestation`, if any, that has a matching `attestation.data`.
     ///
     /// The attestation will be obtained from `self.naive_aggregation_pool`.
diff --git a/beacon_node/beacon_chain/src/block_verification.rs b/beacon_node/beacon_chain/src/block_verification.rs
index 5f8c90d12e8..3b80ca07644 100644
--- a/beacon_node/beacon_chain/src/block_verification.rs
+++ b/beacon_node/beacon_chain/src/block_verification.rs
@@ -1782,7 +1782,12 @@ pub fn check_block_is_finalized_checkpoint_or_descendant<
     fork_choice: &BeaconForkChoice<T>,
     block: B,
 ) -> Result<B, BlockError> {
-    if fork_choice.is_finalized_checkpoint_or_descendant(block.parent_root()) {
+    // If we have a split block newer than finalization then we also ban blocks which are not
+    // descended from that split block.
+    let split = chain.store.get_split_info();
+    if fork_choice.is_finalized_checkpoint_or_descendant(block.parent_root())
+        && fork_choice.is_descendant(split.block_root, block.parent_root())
+    {
         Ok(block)
     } else {
         // If fork choice does *not* consider the parent to be a descendant of the finalized block,
diff --git a/beacon_node/beacon_chain/src/errors.rs b/beacon_node/beacon_chain/src/errors.rs
index 2e13ab4090e..8509c52c8a9 100644
--- a/beacon_node/beacon_chain/src/errors.rs
+++ b/beacon_node/beacon_chain/src/errors.rs
@@ -61,6 +61,7 @@ pub enum BeaconChainError {
     ForkChoiceStoreError(ForkChoiceStoreError),
     MissingBeaconBlock(Hash256),
     MissingBeaconState(Hash256),
+    MissingHotStateSummary(Hash256),
     SlotProcessingError(SlotProcessingError),
     EpochProcessingError(EpochProcessingError),
     StateAdvanceError(StateAdvanceError),
@@ -181,9 +182,9 @@ pub enum BeaconChainError {
         execution_block_hash: Option<ExecutionBlockHash>,
     },
     ForkchoiceUpdate(execution_layer::Error),
-    FinalizedCheckpointMismatch {
-        head_state: Checkpoint,
-        fork_choice: Hash256,
+    InvalidCheckpoint {
+        state_root: Hash256,
+        checkpoint: Checkpoint,
     },
     InvalidSlot(Slot),
     HeadBlockNotFullyVerified {
diff --git a/beacon_node/beacon_chain/src/migrate.rs b/beacon_node/beacon_chain/src/migrate.rs
index bc4b8e1ed86..441237bf168 100644
--- a/beacon_node/beacon_chain/src/migrate.rs
+++ b/beacon_node/beacon_chain/src/migrate.rs
@@ -124,14 +124,22 @@ pub enum Notification {
     Finalization(FinalizationNotification),
     Reconstruction,
     PruneBlobs(Epoch),
+    ManualFinalization(ManualFinalizationNotification),
+}
+
+pub struct ManualFinalizationNotification {
+    pub state_root: BeaconStateHash,
+    pub checkpoint: Checkpoint,
+    pub head_tracker: Arc<HeadTracker>,
+    pub genesis_block_root: Hash256,
 }
 
 pub struct FinalizationNotification {
-    finalized_state_root: BeaconStateHash,
-    finalized_checkpoint: Checkpoint,
-    head_tracker: Arc<HeadTracker>,
-    prev_migration: Arc<Mutex<PrevMigration>>,
-    genesis_block_root: Hash256,
+    pub finalized_state_root: BeaconStateHash,
+    pub finalized_checkpoint: Checkpoint,
+    pub head_tracker: Arc<HeadTracker>,
+    pub prev_migration: Arc<Mutex<PrevMigration>>,
+    pub genesis_block_root: Hash256,
 }
 
 impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Hot, Cold> {
@@ -190,6 +198,14 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
         Ok(())
     }
 
+    pub fn process_manual_finalization(&self, notif: ManualFinalizationNotification) {
+        if let Some(Notification::ManualFinalization(notif)) =
+            self.send_background_notification(Notification::ManualFinalization(notif))
+        {
+            Self::run_manual_migration(self.db.clone(), notif, &self.log);
+        }
+    }
+
     pub fn process_reconstruction(&self) {
         if let Some(Notification::Reconstruction) =
             self.send_background_notification(Notification::Reconstruction)
@@ -289,6 +305,26 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
         }
     }
 
+    fn run_manual_migration(
+        db: Arc<HotColdDB<E, Hot, Cold>>,
+        notif: ManualFinalizationNotification,
+        log: &Logger,
+    ) {
+        // We create a "dummy" prev migration
+        let prev_migration = PrevMigration {
+            epoch: Epoch::new(1),
+            epochs_per_migration: 2,
+        };
+        let notif = FinalizationNotification {
+            finalized_state_root: notif.state_root,
+            finalized_checkpoint: notif.checkpoint,
+            head_tracker: notif.head_tracker,
+            prev_migration: Arc::new(prev_migration.into()),
+            genesis_block_root: notif.genesis_block_root,
+        };
+        Self::run_migration(db, notif, log);
+    }
+
     /// Perform the actual work of `process_finalization`.
     fn run_migration(
         db: Arc<HotColdDB<E, Hot, Cold>>,
@@ -422,16 +458,27 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
             while let Ok(notif) = rx.recv() {
                 let mut reconstruction_notif = None;
                 let mut finalization_notif = None;
+                let mut manual_finalization_notif = None;
                 let mut prune_blobs_notif = None;
                 match notif {
                     Notification::Reconstruction => reconstruction_notif = Some(notif),
                     Notification::Finalization(fin) => finalization_notif = Some(fin),
+                    Notification::ManualFinalization(fin) => manual_finalization_notif = Some(fin),
                     Notification::PruneBlobs(dab) => prune_blobs_notif = Some(dab),
                 }
                 // Read the rest of the messages in the channel, taking the best of each type.
                 for notif in rx.try_iter() {
                     match notif {
                         Notification::Reconstruction => reconstruction_notif = Some(notif),
+                        Notification::ManualFinalization(fin) => {
+                            if let Some(current) = manual_finalization_notif.as_mut() {
+                                if fin.checkpoint.epoch > current.checkpoint.epoch {
+                                    *current = fin;
+                                }
+                            } else {
+                                manual_finalization_notif = Some(fin);
+                            }
+                        }
                         Notification::Finalization(fin) => {
                             if let Some(current) = finalization_notif.as_mut() {
                                 if fin.finalized_checkpoint.epoch
@@ -454,6 +501,9 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
                 if let Some(fin) = finalization_notif {
                     Self::run_migration(db.clone(), fin, &log);
                 }
+                if let Some(fin) = manual_finalization_notif {
+                    Self::run_manual_migration(db.clone(), fin, &log);
+                }
                 if let Some(dab) = prune_blobs_notif {
                     Self::run_prune_blobs(db.clone(), dab, &log);
                 }
diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs
index 38758fe5c31..d0e257ce7e5 100644
--- a/beacon_node/http_api/src/lib.rs
+++ b/beacon_node/http_api/src/lib.rs
@@ -86,11 +86,11 @@ use tokio_stream::{
 };
 use types::{
     fork_versioned_response::EmptyMetadata, Attestation, AttestationData, AttestationShufflingId,
-    AttesterSlashing, BeaconStateError, ChainSpec, CommitteeCache, ConfigAndPreset, Epoch, EthSpec,
-    ForkName, ForkVersionedResponse, Hash256, ProposerPreparationData, ProposerSlashing,
-    RelativeEpoch, SignedAggregateAndProof, SignedBlindedBeaconBlock, SignedBlsToExecutionChange,
-    SignedContributionAndProof, SignedValidatorRegistrationData, SignedVoluntaryExit, Slot,
-    SyncCommitteeMessage, SyncContributionData,
+    AttesterSlashing, BeaconStateError, ChainSpec, Checkpoint, CommitteeCache, ConfigAndPreset,
+    Epoch, EthSpec, ForkName, ForkVersionedResponse, Hash256, ProposerPreparationData,
+    ProposerSlashing, RelativeEpoch, SignedAggregateAndProof, SignedBlindedBeaconBlock,
+    SignedBlsToExecutionChange, SignedContributionAndProof, SignedValidatorRegistrationData,
+    SignedVoluntaryExit, Slot, SyncCommitteeMessage, SyncContributionData,
 };
 use validator::pubkey_to_validator_index;
 use version::{
@@ -4133,6 +4133,35 @@ pub fn serve<T: BeaconChainTypes>(
             },
         );
 
+    // POST lighthouse/finalize
+    let post_lighthouse_finalize = warp::path("lighthouse")
+        .and(warp::path("finalize"))
+        .and(warp::path::end())
+        .and(warp_utils::json::json())
+        .and(task_spawner_filter.clone())
+        .and(chain_filter.clone())
+        .then(
+            |request_data: api_types::ManualFinalizationRequestData,
+             task_spawner: TaskSpawner<T::EthSpec>,
+             chain: Arc<BeaconChain<T>>| {
+                task_spawner.blocking_json_task(Priority::P0, move || {
+                    let checkpoint = Checkpoint {
+                        epoch: request_data.epoch,
+                        root: request_data.block_root,
+                    };
+
+                    chain
+                        .manually_finalize_state(request_data.state_root, checkpoint)
+                        .map(|_| api_types::GenericResponse::from(request_data))
+                        .map_err(|e| {
+                            warp_utils::reject::custom_bad_request(format!(
+                                "Failed to finalize state due to error: {e:?}"
+                            ))
+                        })
+                })
+            },
+        );
+
     // POST lighthouse/liveness
     let post_lighthouse_liveness = warp::path("lighthouse")
         .and(warp::path("liveness"))
@@ -4932,6 +4961,7 @@ pub fn serve<T: BeaconChainTypes>(
                     .uor(post_lighthouse_block_rewards)
                     .uor(post_lighthouse_ui_validator_metrics)
                     .uor(post_lighthouse_ui_validator_info)
+                    .uor(post_lighthouse_finalize)
                     .recover(warp_utils::reject::handle_rejection),
             ),
         )
diff --git a/beacon_node/network/src/network_beacon_processor/sync_methods.rs b/beacon_node/network/src/network_beacon_processor/sync_methods.rs
index ef4d9cc4c16..64e69f8f4ed 100644
--- a/beacon_node/network/src/network_beacon_processor/sync_methods.rs
+++ b/beacon_node/network/src/network_beacon_processor/sync_methods.rs
@@ -740,6 +740,19 @@ impl<T: BeaconChainTypes> NetworkBeaconProcessor<T> {
                 debug!(self.log, "Finalized or earlier block processed";);
                 Ok(())
             }
+            BlockError::NotFinalizedDescendant { block_parent_root } => {
+                debug!(
+                    self.log,
+                    "Not syncing to a chain that conflicts with the canonical or manual finalized checkpoint"
+                );
+                Err(ChainSegmentFailed {
+                    message: format!(
+                        "Block with parent_root {} conflicts with our checkpoint state",
+                        block_parent_root
+                    ),
+                    peer_action: Some(PeerAction::Fatal),
+                })
+            }
             BlockError::GenesisBlock => {
                 debug!(self.log, "Genesis block was processed");
                 Ok(())
diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs
index 59374f629d6..7d70d242be7 100644
--- a/common/eth2/src/types.rs
+++ b/common/eth2/src/types.rs
@@ -1424,6 +1424,13 @@ pub struct StandardLivenessResponseData {
     pub is_live: bool,
 }
 
+#[derive(Debug, Serialize, Deserialize)]
+pub struct ManualFinalizationRequestData {
+    pub state_root: Hash256,
+    pub epoch: Epoch,
+    pub block_root: Hash256,
+}
+
 #[derive(Debug, Serialize, Deserialize)]
 pub struct LivenessRequestData {
     pub epoch: Epoch,
diff --git a/consensus/fork_choice/src/fork_choice.rs b/consensus/fork_choice/src/fork_choice.rs
index 4c25be950b0..ddd5923849c 100644
--- a/consensus/fork_choice/src/fork_choice.rs
+++ b/consensus/fork_choice/src/fork_choice.rs
@@ -1255,6 +1255,11 @@ where
             .is_finalized_checkpoint_or_descendant::<E>(block_root)
     }
 
+    pub fn is_descendant(&self, ancestor_root: Hash256, descendant_root: Hash256) -> bool {
+        self.proto_array
+            .is_descendant(ancestor_root, descendant_root)
+    }
+
     /// Returns `Ok(true)` if `block_root` has been imported optimistically or deemed invalid.
     ///
     /// Returns `Ok(false)` if `block_root`'s execution payload has been elected as fully VALID, if


Reviews:
user: login: michaelsproul
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-02-28T22:01:08Z
user: login: dapplion
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-01T01:41:19Z
user: login: michaelsproul
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-02T22:48:43Z
user: login: jimmygchen
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-03T04:43:39Z
user: login: michaelsproul
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-03T05:28:23Z
user: login: michaelsproul
state: APPROVED
author_association: MEMBER
submitted_at: 2025-03-03T05:31:06Z

Review comments:
diff_hunk: @@ -1707,6 +1707,16 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         }
     }
 
+    pub fn manually_finalize_state(&self, state_root: Hash256, checkpoint: Checkpoint) {
+        let notif = ManualFinalizationNotification {
path: beacon_node/beacon_chain/src/beacon_chain.rs
user: login: dapplion
body: Here we should sanity check that this checkpoint is an actual checkpoint to not mess up the DB too much
created_at: 2025-02-28T21:00:47Z
author_association: COLLABORATOR
path: testing/.DS_Store
user: login: michaelsproul
body: can delete
created_at: 2025-02-28T22:01:08Z
author_association: MEMBER
diff_hunk: @@ -1707,6 +1707,16 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         }
     }
 
+    pub fn manually_finalize_state(&self, state_root: Hash256, checkpoint: Checkpoint) {
+        let notif = ManualFinalizationNotification {
path: beacon_node/beacon_chain/src/beacon_chain.rs
user: login: michaelsproul
body: 100%

I think we should load the `HotStateSummary` for the `state_root` and check the provided `slot` and `block_root` against it (should match `slot` and `latest_block_root` from the summary).
created_at: 2025-03-02T22:48:43Z
author_association: MEMBER
diff_hunk: @@ -190,6 +198,10 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
         Ok(())
     }
 
+    pub fn process_manual_finalization(&self, notif: ManualFinalizationNotification) {
+        let _ = self.send_background_notification(Notification::ManualFinalization(notif));
path: beacon_node/beacon_chain/src/migrate.rs
user: login: jimmygchen
body: I don't think we ever run store migrator in the foreground other than testing, but probably should support it given we do it  for all 3 other notifications, and if we want to test this with `BeaconChainHarness`, it won't be immediately obvious why this doesn't get run.
created_at: 2025-03-03T04:43:39Z
author_association: MEMBER
diff_hunk: @@ -190,6 +198,10 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
         Ok(())
     }
 
+    pub fn process_manual_finalization(&self, notif: ManualFinalizationNotification) {
+        let _ = self.send_background_notification(Notification::ManualFinalization(notif));
path: beacon_node/beacon_chain/src/migrate.rs
user: login: michaelsproul
body: Good point. We should do this for the clean version of this PR
created_at: 2025-03-03T05:28:23Z
author_association: MEMBER

PR comments:
user: login: eserilev
created_at: 2025-03-01T02:59:22Z
body: one quick and dirty thing we might be able to try 
```rust
// add some for loop here so we only fetch at most one state per roots iterator
for {
    let state_roots = RootsIterator::new(&store, finalized_state)
        .take_while(|result| match result {
            Ok((_, _, slot)) => *slot >= current_split_slot,
            Err(_) => true,
        })
        .collect::<Result<Vec<_>, _>>()?;
    // do the hot db -> cold db migration + pruning
 }
  ```
author_association: MEMBER
user: login: michaelsproul
created_at: 2025-03-01T09:56:30Z
body: I think the `RootsIterator` is _already_ inside a loop, so there should only be one at a time:

https://github.com/sigp/lighthouse/blob/404e48d740778181ba18ce2803562fc9f8c4f19b/beacon_node/beacon_chain/src/migrate.rs#L602-L630

Similarly the `RootsIterator` itself jumps back over states one at a time too.

The only thing I can think is that we're putting these states in the cache, and that's enough to cause an OOM with `--state-cache-size 32`
author_association: MEMBER