Repository: op-rs/kona
Pull request #1675: feat(node/service): Flush Channel on Invalid Payloads
Author: login: refcell
Labels: name: K-feature
description: Kind: feature
name: A-node
description: Area: cl node (eq. Go op-node) handles single-chain consensus
Created: 2025-05-07T21:40:37Z | Merged: 2025-05-08T20:46:05Z

### Description

Wires up the engine actor to send a signal to the derivation actor.

This is necessary when the engine api marks a payload `INVALID`.
Post-Holocene, the payload becomes deposit-only and is re-executed.
At the same time, the derivation pipeline needs to be flushed, so
the channel and remaining buffered batches are cleared.

Makes progress on #1239


Diff:
diff --git a/crates/node/engine/src/task_queue/tasks/build/error.rs b/crates/node/engine/src/task_queue/tasks/build/error.rs
index 7c71519bd4..7661ed9aa4 100644
--- a/crates/node/engine/src/task_queue/tasks/build/error.rs
+++ b/crates/node/engine/src/task_queue/tasks/build/error.rs
@@ -38,6 +38,11 @@ pub enum BuildTaskError {
     /// A deposit-only payload failed to import.
     #[error("Deposit-only payload failed to import")]
     DepositOnlyPayloadFailed,
+
+    /// The payload is invalid, and the derivation pipeline must
+    /// be flushed post-holocene.
+    #[error("Invalid payload, must flush post-holocene")]
+    HoloceneInvalidFlush,
 }
 
 impl From<BuildTaskError> for EngineTaskError {
@@ -51,6 +56,7 @@ impl From<BuildTaskError> for EngineTaskError {
             BuildTaskError::GetPayloadFailed(_) => Self::Temporary(Box::new(value)),
             BuildTaskError::NewPayloadFailed(_) => Self::Temporary(Box::new(value)),
             BuildTaskError::InvalidForkchoiceState => Self::Reset(Box::new(value)),
+            BuildTaskError::HoloceneInvalidFlush => Self::Flush(Box::new(value)),
             BuildTaskError::FinalizedAheadOfUnsafe(_, _) => Self::Critical(Box::new(value)),
             BuildTaskError::DepositOnlyPayloadFailed => Self::Critical(Box::new(value)),
         }
diff --git a/crates/node/engine/src/task_queue/tasks/build/task.rs b/crates/node/engine/src/task_queue/tasks/build/task.rs
index 0e8123ca07..31360378da 100644
--- a/crates/node/engine/src/task_queue/tasks/build/task.rs
+++ b/crates/node/engine/src/task_queue/tasks/build/task.rs
@@ -245,7 +245,8 @@ impl BuildTask {
                 } else {
                     warn!(target: "engine_builder", "Payload import failed: {validation_error}");
                     warn!(target: "engine_builder", "Re-attempting payload import with deposits only.");
-                    unimplemented!("HOLOCENE: Re-attempt payload import with deposits only");
+                    // HOLOCENE: Re-attempt payload import with deposits only
+                    Err(BuildTaskError::HoloceneInvalidFlush)
                 }
             }
             s => {
diff --git a/crates/node/engine/src/task_queue/tasks/task.rs b/crates/node/engine/src/task_queue/tasks/task.rs
index b251702141..e82d2342ed 100644
--- a/crates/node/engine/src/task_queue/tasks/task.rs
+++ b/crates/node/engine/src/task_queue/tasks/task.rs
@@ -92,6 +92,10 @@ impl EngineTaskExt for EngineTask {
                     warn!(target: "engine", "Engine requested derivation reset");
                     return Err(EngineTaskError::Reset(e));
                 }
+                EngineTaskError::Flush(e) => {
+                    warn!(target: "engine", "Engine requested derivation flush");
+                    return Err(EngineTaskError::Flush(e));
+                }
             }
         }
 
@@ -118,4 +122,7 @@ pub enum EngineTaskError {
     /// An error that requires a derivation pipeline reset.
     #[error("Derivation pipeline reset required: {0}")]
     Reset(Box<dyn std::error::Error>),
+    /// An error that requires the derivation pipeline to be flushed.
+    #[error("Derivation pipeline flush required: {0}")]
+    Flush(Box<dyn std::error::Error>),
 }
diff --git a/crates/node/service/src/actors/derivation.rs b/crates/node/service/src/actors/derivation.rs
index 785167cab1..9bb4d64b57 100644
--- a/crates/node/service/src/actors/derivation.rs
+++ b/crates/node/service/src/actors/derivation.rs
@@ -5,7 +5,7 @@ use async_trait::async_trait;
 use kona_derive::{
     errors::{PipelineError, PipelineErrorKind, ResetError},
     traits::{Pipeline, SignalReceiver},
-    types::{ActivationSignal, ResetSignal, StepResult},
+    types::{ActivationSignal, ResetSignal, Signal, StepResult},
 };
 use kona_protocol::{BlockInfo, L2BlockInfo, OpAttributesWithParent};
 use thiserror::Error;
@@ -31,10 +31,29 @@ where
     l2_safe_head: L2BlockInfo,
     /// A receiver that tells derivation to begin.
     sync_complete_rx: UnboundedReceiver<()>,
+    /// A receiver that sends a [`Signal`] to the derivation pipeline.
+    ///
+    /// The derivation actor steps over the derivation pipeline to generate
+    /// [`OpAttributesWithParent`]. These attributes then need to be executed
+    /// via the engine api, which is done by sending them through the
+    /// [`Self::attributes_out`] channel.
+    ///
+    /// When the engine api receives an `INVALID` response for a new block (
+    /// the new [`OpAttributesWithParent`]) during block building, the payload
+    /// is reduced to "deposits-only". When this happens, the channel and
+    /// remaining buffered batches need to be flushed out of the derivation
+    /// pipeline.
+    ///
+    /// This channel allows the engine to send a [`Signal::FlushChannel`]
+    /// message back to the derivation pipeline when an `INVALID` response
+    /// occurs.
+    ///
+    /// Specs: <https://specs.optimism.io/protocol/derivation.html#l1-sync-payload-attributes-processing>
+    derivation_signal_rx: UnboundedReceiver<Signal>,
     /// A flag indicating whether the derivation pipeline is ready to start.
     engine_ready: bool,
     /// The sender for derived [OpAttributesWithParent]s produced by the actor.
-    attributes_out: UnboundedSender<OpAttributesWithParent>,
+    pub attributes_out: UnboundedSender<OpAttributesWithParent>,
     /// The receiver for L1 head update notifications.
     l1_head_updates: UnboundedReceiver<BlockInfo>,
     /// The cancellation token, shared between all tasks.
@@ -50,6 +69,7 @@ where
         pipeline: P,
         l2_safe_head: L2BlockInfo,
         sync_complete_rx: UnboundedReceiver<()>,
+        derivation_signal_rx: UnboundedReceiver<Signal>,
         attributes_out: UnboundedSender<OpAttributesWithParent>,
         l1_head_updates: UnboundedReceiver<BlockInfo>,
         cancellation: CancellationToken,
@@ -58,6 +78,7 @@ where
             pipeline,
             l2_safe_head,
             sync_complete_rx,
+            derivation_signal_rx,
             engine_ready: false,
             attributes_out,
             l1_head_updates,
@@ -65,6 +86,16 @@ where
         }
     }
 
+    /// Handles a [`Signal`] received over the derivation signal receiver channel.
+    async fn signal(&mut self, signal: Signal) {
+        match self.pipeline.signal(signal).await {
+            Ok(_) => info!(target: "derivation", ?signal, "[SIGNAL] Executed Successfully"),
+            Err(e) => {
+                error!(target: "derivation", ?e, ?signal, "Failed to signal derivation pipeline")
+            }
+        }
+    }
+
     /// Attempts to step the derivation pipeline forward as much as possible in order to produce the
     /// next safe payload.
     async fn produce_next_safe_payload(
@@ -203,6 +234,18 @@ where
 
                     self.process(InboundDerivationMessage::NewDataAvailable).await?;
                 }
+                signal = self.derivation_signal_rx.recv() => {
+                    let Some(signal) = signal else {
+                        error!(
+                            target: "derivation",
+                            ?signal,
+                            "DerivationActor failed to receive signal"
+                        );
+                        return Err(DerivationError::SignalReceiveFailed);
+                    };
+
+                    self.signal(signal).await;
+                }
             }
         }
     }
@@ -251,4 +294,7 @@ pub enum DerivationError {
     /// An error originating from the broadcast sender.
     #[error("Failed to send event to broadcast sender")]
     Sender(#[from] Box<SendError<OpAttributesWithParent>>),
+    /// An error from the signal receiver.
+    #[error("Failed to receive signal")]
+    SignalReceiveFailed,
 }
diff --git a/crates/node/service/src/actors/engine.rs b/crates/node/service/src/actors/engine.rs
index 9a80116550..329bfc3547 100644
--- a/crates/node/service/src/actors/engine.rs
+++ b/crates/node/service/src/actors/engine.rs
@@ -2,9 +2,10 @@
 
 use alloy_rpc_types_engine::JwtSecret;
 use async_trait::async_trait;
+use kona_derive::types::Signal;
 use kona_engine::{
     ConsolidateTask, Engine, EngineClient, EngineStateBuilder, EngineStateBuilderError, EngineTask,
-    InsertUnsafeTask,
+    EngineTaskError, InsertUnsafeTask,
 };
 use kona_genesis::RollupConfig;
 use kona_protocol::OpAttributesWithParent;
@@ -33,6 +34,9 @@ pub struct EngineActor {
     /// A channel to send a signal that syncing is complete.
     /// Informs the derivation actor to start.
     sync_complete_tx: UnboundedSender<()>,
+    /// A way for the engine actor to signal back to the derivation actor
+    /// if a block building task produced an `INVALID` response.
+    derivation_signal_tx: UnboundedSender<Signal>,
     /// A channel to receive [`RuntimeConfig`] from the runtime actor.
     runtime_config_rx: UnboundedReceiver<RuntimeConfig>,
     /// A channel to receive [`OpAttributesWithParent`] from the derivation actor.
@@ -51,6 +55,7 @@ impl EngineActor {
         client: EngineClient,
         engine: Engine,
         sync_complete_tx: UnboundedSender<()>,
+        derivation_signal_tx: UnboundedSender<Signal>,
         runtime_config_rx: UnboundedReceiver<RuntimeConfig>,
         attributes_rx: UnboundedReceiver<OpAttributesWithParent>,
         unsafe_block_rx: UnboundedReceiver<OpNetworkPayloadEnvelope>,
@@ -60,6 +65,7 @@ impl EngineActor {
             config,
             client: Arc::new(client),
             sync_complete_tx,
+            derivation_signal_tx,
             engine,
             runtime_config_rx,
             attributes_rx,
@@ -143,9 +149,24 @@ impl NodeActor for EngineActor {
                     return Ok(());
                 }
                 res = self.engine.drain() => {
-                    if let Err(e) = res {
-                        warn!(target: "engine", "Encountered error draining engine api tasks: {:?}", e);
-                    }
+                    match res {
+                        Ok(_) => trace!(target: "engine", "[ENGINE] tasks drained"),
+                        Err(EngineTaskError::Flush(e)) => {
+                            // This error is encountered when the payload is marked INVALID
+                            // by the engine api. Post-holocene, the payload is replaced by
+                            // a "deposits-only" block and re-executed. At the same time,
+                            // the channel and any remaining buffered batches are flushed.
+                            warn!(target: "engine", ?e, "[HOLOCENE] Invalid payload, Flushing derivation pipeline.");
+                            match self.derivation_signal_tx.send(Signal::FlushChannel) {
+                                Ok(_) => debug!(target: "engine", "[SENT] flush signal to derivation actor"),
+                                Err(e) => {
+                                    error!(target: "engine", ?e, "[ENGINE] Failed to send flush signal to the derivation actor.");
+                                    self.cancellation.cancel();
+                                    return Err(EngineError::ChannelClosed);
+                                }
+                            }
+                        }
+                        Err(e) => warn!(target: "engine", ?e, "Error draining engine tasks"),                    }
                 }
                 attributes = self.attributes_rx.recv() => {
                     let Some(attributes) = attributes else {
diff --git a/crates/node/service/src/service/validator.rs b/crates/node/service/src/service/validator.rs
index 5ba016b587..9c434e4be5 100644
--- a/crates/node/service/src/service/validator.rs
+++ b/crates/node/service/src/service/validator.rs
@@ -102,6 +102,7 @@ pub trait ValidatorNodeService {
         let (unsafe_block_tx, unsafe_block_rx) = mpsc::unbounded_channel();
         let (sync_complete_tx, sync_complete_rx) = mpsc::unbounded_channel();
         let (runtime_config_tx, runtime_config_rx) = mpsc::unbounded_channel();
+        let (derivation_signal_tx, derivation_signal_rx) = mpsc::unbounded_channel();
 
         let (block_signer_tx, block_signer_rx) = mpsc::unbounded_channel();
         let da_watcher =
@@ -112,6 +113,7 @@ pub trait ValidatorNodeService {
             derivation_pipeline,
             l2_forkchoice_state.safe,
             sync_complete_rx,
+            derivation_signal_rx,
             derived_payload_tx,
             new_head_rx,
             cancellation.clone(),
@@ -132,6 +134,7 @@ pub trait ValidatorNodeService {
             client,
             engine,
             sync_complete_tx,
+            derivation_signal_tx,
             runtime_config_rx,
             derived_payload_rx,
             unsafe_block_rx,


Reviews:
user: login: copilot-pull-request-reviewer[bot]
body: ## Pull Request Overview

This PR adds a channel between the engine and derivation actors to support flushing the derivation pipeline when an invalid payload is detected. Key changes include:
- Adding derivation_signal channels in the node service and actor modules.
- Implementing flush handling in the engine actor by sending a Signal::FlushChannel.
- Updating task queue error handling to incorporate a new Flush error type.

### Reviewed Changes

Copilot reviewed 6 out of 6 changed files in this pull request and generated no comments.

<details>
<summary>Show a summary per file</summary>

| File | Description |
| ---- | ----------- |
| crates/node/service/src/service/validator.rs | Added channel for derivation_signal integration. |
| crates/node/service/src/actors/engine.rs | Injected derivation_signal_tx and updated error handling for flush actions in engine tasks. |
| crates/node/service/src/actors/derivation.rs | Added derivation_signal_rx to the derivation actor and handling for flush signals. |
| crates/node/engine/src/task_queue/tasks/task.rs | Introduced flush error branch for engine tasks. |
| crates/node/engine/src/task_queue/tasks/build/task.rs | Updated build task error handling to return a flush error instead of panicking. |
| crates/node/engine/src/task_queue/tasks/build/error.rs | Added a new enum variant and conversion for HoloceneInvalidFlush. |
</details>



<details>
<summary>Comments suppressed due to low confidence (1)</summary>

**crates/node/service/src/actors/derivation.rs:238**
* [nitpick] Since the signal variable is None in this branch, logging '?signal' is redundant and could lead to confusion. Consider removing '?signal' from the log to keep the message clear.
```
error!(target: "derivation", ?signal, "DerivationActor failed to receive signal");
```
</details>


state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-07T21:40:52Z
user: login: theochap
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-07T22:12:28Z
user: login: refcell
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-08T14:26:16Z
user: login: theochap
state: APPROVED
author_association: MEMBER
submitted_at: 2025-05-08T19:14:36Z

Review comments:
diff_hunk: @@ -202,6 +233,18 @@ where
 
                     self.process(InboundDerivationMessage::NewDataAvailable).await?;
                 }
+                signal = self.derivation_signal_rx.recv() => {
+                    let Some(signal) = signal else {
+                        error!(
+                            target: "derivation",
+                            ?signal,
+                            "DerivationActor failed to receive signal"
+                        );
+                        return Ok(());
path: crates/node/service/src/actors/derivation.rs
user: login: theochap
body: Should we return an error in that case (or the one above)?
created_at: 2025-05-07T22:12:27Z
author_association: MEMBER
diff_hunk: @@ -202,6 +233,18 @@ where
 
                     self.process(InboundDerivationMessage::NewDataAvailable).await?;
                 }
+                signal = self.derivation_signal_rx.recv() => {
+                    let Some(signal) = signal else {
+                        error!(
+                            target: "derivation",
+                            ?signal,
+                            "DerivationActor failed to receive signal"
+                        );
+                        return Ok(());
path: crates/node/service/src/actors/derivation.rs
user: login: refcell
body: I think you're right we should error here since it's critical to consensus
created_at: 2025-05-08T14:26:16Z
author_association: CONTRIBUTOR

PR comments:
user: login: refcell
created_at: 2025-05-07T21:40:40Z
body: ## 📚 $\text{Stack Overview}$

Pulls submitted in this stack:
* #1676
* #1675 👈
* `main`

_This comment was automatically generated by [`st`](https://github.com/clabby/st)._
author_association: CONTRIBUTOR
user: login: codecov[bot]
created_at: 2025-05-07T21:44:01Z
body: ## [Codecov](https://app.codecov.io/gh/op-rs/kona/pull/1675?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=op-rs) Report
All modified and coverable lines are covered by tests :white_check_mark:
> Project coverage is 92.5%. Comparing base [(`4d39a83`)](https://app.codecov.io/gh/op-rs/kona/commit/4d39a83cde77e8dfda8f2d3640b736c2923f5b54?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=op-rs) to head [(`b8dbb1e`)](https://app.codecov.io/gh/op-rs/kona/commit/b8dbb1e486ecea806531a6d34efc87ee34671b90?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=op-rs).
> Report is 1 commits behind head on main.

:white_check_mark: All tests successful. No failed tests found.


[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/op-rs/kona/pull/1675?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=op-rs).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=op-rs).

<details><summary> :rocket: New features to boost your workflow: </summary>

- :package: [JS Bundle Analysis](https://docs.codecov.com/docs/javascript-bundle-analysis): Save yourself from yourself by tracking and limiting bundle sizes in JS merges.
</details>
author_association: NONE