Repository: hyperlane-xyz/hyperlane-monorepo
Pull request #6350: feat: separate tx id indexing and cursor indexing into separate threads
Author: login: kamiyaa
Created: 2025-05-26T17:25:28Z | Merged: 2025-05-29T13:48:58Z

### Description
Currently, the watermark cursor indexing shares the same loop as the message tx id indexing.
We rate-limit ourselves for the watermark cursor by having sleep() inside its logic to prevent being rate limited by RPCs.
The side effect of this is we also rate limit the message tx id indexing.

#### Solution:

Separate these indexing into separate threads

 - updated liveliness metrics to include which task failed
 - updated tx id thread to wait on recv()
   - this might cause liveliness metric to false report

### Drive-bys
 - disable tx id broadcasting for igp and merkle tree insertion for relayer
   - because we only use the message contract sync broadcasting

### Related issues

 - Fixes https://linear.app/hyperlane-xyz/issue/ENG-1605/igp-indexing-is-up-to-30s-behind-other-indexed-items

### Backward compatibility

Yes

### Testing
 - E2E tests


Diff:
diff --git a/rust/main/agents/relayer/src/relayer.rs b/rust/main/agents/relayer/src/relayer.rs
index 343bb5b3740..49bbc86a82e 100644
--- a/rust/main/agents/relayer/src/relayer.rs
+++ b/rust/main/agents/relayer/src/relayer.rs
@@ -66,6 +66,7 @@ use crate::{processor::Processor, server::ENDPOINT_MESSAGES_QUEUE_SIZE};
 
 const CURSOR_BUILDING_ERROR: &str = "Error building cursor for origin";
 const CURSOR_INSTANTIATION_ATTEMPTS: usize = 10;
+const ADVANCED_LOG_META: bool = false;
 
 #[derive(Debug, Hash, PartialEq, Eq, Copy, Clone)]
 struct ContextKey {
@@ -225,7 +226,7 @@ impl BaseAgent for Relayer {
                 dbs.iter()
                     .map(|(d, db)| (d.clone(), Arc::new(db.clone())))
                     .collect(),
-                false,
+                ADVANCED_LOG_META,
                 settings.tx_id_indexing_enabled,
             )
             .await?
@@ -245,8 +246,8 @@ impl BaseAgent for Relayer {
                         dbs.iter()
                             .map(|(d, db)| (d.clone(), Arc::new(db.clone())))
                             .collect(),
+                        ADVANCED_LOG_META,
                         false,
-                        settings.tx_id_indexing_enabled,
                     )
                     .await?
                     .into_iter()
@@ -267,8 +268,8 @@ impl BaseAgent for Relayer {
                 dbs.iter()
                     .map(|(d, db)| (d.clone(), Arc::new(db.clone())))
                     .collect(),
+                ADVANCED_LOG_META,
                 false,
-                settings.tx_id_indexing_enabled,
             )
             .await?
             .into_iter()
diff --git a/rust/main/hyperlane-base/src/contract_sync/metrics.rs b/rust/main/hyperlane-base/src/contract_sync/metrics.rs
index 7c906b28f10..3b684e0ba8d 100644
--- a/rust/main/hyperlane-base/src/contract_sync/metrics.rs
+++ b/rust/main/hyperlane-base/src/contract_sync/metrics.rs
@@ -56,7 +56,7 @@ impl ContractSyncMetrics {
             .new_int_gauge(
                 "contract_sync_liveness",
                 "Last timestamp observed by contract sync",
-                &["data_type", "chain"],
+                &["data_type", "chain", "task"],
             )
             .expect("failed to register liveness metric");
 
diff --git a/rust/main/hyperlane-base/src/contract_sync/mod.rs b/rust/main/hyperlane-base/src/contract_sync/mod.rs
index b33067b633a..8ce652e8f02 100644
--- a/rust/main/hyperlane-base/src/contract_sync/mod.rs
+++ b/rust/main/hyperlane-base/src/contract_sync/mod.rs
@@ -8,17 +8,17 @@ use broadcast::BroadcastMpscSender;
 use cursors::*;
 use derive_new::new;
 use eyre::Result;
+use prometheus::core::{AtomicI64, AtomicU64, GenericCounter, GenericGauge};
+use tokio::sync::{mpsc::Receiver as MpscReceiver, Mutex};
+use tokio::time::sleep;
+use tracing::{debug, info, instrument, trace, warn};
+
 use hyperlane_core::{
     utils::fmt_sync_time, ContractSyncCursor, CursorAction, HyperlaneDomain, HyperlaneLogStore,
     HyperlaneSequenceAwareIndexerStore, HyperlaneWatermarkedLogStore, Indexer,
     SequenceAwareIndexer,
 };
 use hyperlane_core::{Indexed, LogMeta, H512};
-pub use metrics::ContractSyncMetrics;
-use prometheus::core::{AtomicI64, AtomicU64, GenericCounter, GenericGauge};
-use tokio::sync::mpsc::{error::TryRecvError, Receiver as MpscReceiver};
-use tokio::time::sleep;
-use tracing::{debug, info, instrument, trace, warn};
 
 use crate::settings::IndexSettings;
 
@@ -28,6 +28,8 @@ pub(crate) mod cursors;
 mod eta_calculator;
 mod metrics;
 
+pub use metrics::ContractSyncMetrics;
+
 use cursors::ForwardBackwardSequenceAwareSyncCursor;
 
 const SLEEP_DURATION: Duration = Duration::from_secs(5);
@@ -81,8 +83,8 @@ impl<T: Indexable, S: HyperlaneLogStore<T>, I: Indexer<T>> ContractSync<T, S, I>
 impl<T, S, I> ContractSync<T, S, I>
 where
     T: Indexable + Debug + Send + Sync + Clone + Eq + Hash + 'static,
-    S: HyperlaneLogStore<T>,
-    I: Indexer<T> + 'static,
+    S: HyperlaneLogStore<T> + Clone + 'static,
+    I: Indexer<T> + Clone + 'static,
 {
     /// The domain that this ContractSync is running on
     pub fn domain(&self) -> &HyperlaneDomain {
@@ -95,7 +97,7 @@ where
 
     /// Sync logs and write them to the LogStore
     #[instrument(name = "ContractSync", fields(domain=self.domain().name()), skip(self, opts))]
-    pub async fn sync(&self, label: &'static str, mut opts: SyncOptions<T>) {
+    pub async fn sync(&self, label: &'static str, opts: SyncOptions<T>) {
         let chain_name = self.domain.as_ref();
         let indexed_height_metric = self
             .metrics
@@ -105,31 +107,74 @@ where
             .metrics
             .stored_events
             .with_label_values(&[label, chain_name]);
-        let liveness_metric = self
-            .metrics
-            .liveness_metrics
-            .with_label_values(&[label, chain_name]);
 
-        loop {
-            Self::update_liveness_metric(&liveness_metric);
-            if let Some(rx) = opts.tx_id_receiver.as_mut() {
-                self.fetch_logs_from_receiver(rx, &stored_logs_metric).await;
-            }
-            if let Some(cursor) = opts.cursor.as_mut() {
-                self.fetch_logs_with_cursor(cursor, &stored_logs_metric, &indexed_height_metric)
+        // need to put this behind an Arc Mutex because we might
+        // index the same event twice now. Which causes e2e to fail
+        let shared_store = Arc::new(Mutex::new(self.store.clone()));
+
+        // transaction id task for fetching events via transaction id
+        let tx_id_task = match opts.tx_id_receiver {
+            Some(rx) => {
+                let liveness_metric = self.metrics.liveness_metrics.with_label_values(&[
+                    label,
+                    chain_name,
+                    "tx_id_task",
+                ]);
+                let domain_clone = self.domain.clone();
+                let indexer_clone = self.indexer.clone();
+                let store_clone = shared_store.clone();
+                let stored_logs_metric = stored_logs_metric.clone();
+                tokio::task::spawn(async move {
+                    Self::tx_id_indexer_task(
+                        domain_clone,
+                        indexer_clone,
+                        store_clone,
+                        rx,
+                        stored_logs_metric,
+                        liveness_metric,
+                    )
                     .await;
+                })
             }
+            None => tokio::task::spawn(async {}),
+        };
 
-            // Added so that we confuse compiler that it is an infinite loop
-            if false {
-                break;
+        // cursor task for fetching events via range querying
+        let cursor_task = match opts.cursor {
+            Some(cursor) => {
+                let liveness_metric = self.metrics.liveness_metrics.with_label_values(&[
+                    label,
+                    chain_name,
+                    "cursor_task",
+                ]);
+                let domain_clone = self.domain.clone();
+                let indexer_clone = self.indexer.clone();
+                let store_clone = shared_store.clone();
+                let broadcast_sender = self.broadcast_sender.clone();
+
+                let stored_logs_metric = stored_logs_metric.clone();
+
+                tokio::task::spawn(async {
+                    Self::cursor_indexer_task(
+                        domain_clone,
+                        indexer_clone,
+                        store_clone,
+                        cursor,
+                        broadcast_sender,
+                        stored_logs_metric,
+                        indexed_height_metric,
+                        liveness_metric,
+                    )
+                    .await;
+                })
             }
-        }
+            None => tokio::task::spawn(async {}),
+        };
 
-        // Although the above loop should never end (unless by panicking),
-        // we put log here to make sure that we see when this method returns normally.
-        // Hopefully, compiler will not optimise this code out.
-        info!(chain = chain_name, label, "contract sync loop exit");
+        let res = tokio::join!(tx_id_task, cursor_task);
+
+        // we should never reach this because the 2 tasks should never end
+        tracing::error!(chain = chain_name, label, ?res, "contract sync loop exit");
     }
 
     fn update_liveness_metric(liveness_metric: &GenericGauge<AtomicI64>) {
@@ -141,120 +186,133 @@ where
         );
     }
 
-    #[instrument(fields(domain=self.domain().name()), skip(self, recv, stored_logs_metric))]
-    async fn fetch_logs_from_receiver(
-        &self,
-        recv: &mut MpscReceiver<H512>,
-        stored_logs_metric: &GenericCounter<AtomicU64>,
+    #[instrument(fields(domain=domain.name()), skip(indexer, store, recv, stored_logs_metric, liveness_metric))]
+    async fn tx_id_indexer_task(
+        domain: HyperlaneDomain,
+        indexer: I,
+        store: Arc<Mutex<S>>,
+        mut recv: MpscReceiver<H512>,
+        stored_logs_metric: GenericCounter<AtomicU64>,
+        liveness_metric: GenericGauge<AtomicI64>,
     ) {
         loop {
-            match recv.try_recv() {
-                Ok(tx_id) => {
-                    let logs = match self.indexer.fetch_logs_by_tx_hash(tx_id).await {
-                        Ok(logs) => logs,
-                        Err(err) => {
-                            warn!(?err, ?tx_id, "Error fetching logs for tx id");
-                            continue;
-                        }
-                    };
-                    let logs = self.dedupe_and_store_logs(logs, stored_logs_metric).await;
-                    let num_logs = logs.len() as u64;
-                    info!(
-                        num_logs,
-                        ?tx_id,
-                        sequences = ?logs.iter().map(|(log, _)| log.sequence).collect::<Vec<_>>(),
-                        pending_ids = ?recv.len(),
-                        "Found log(s) for tx id"
-                    );
-                }
-                Err(TryRecvError::Empty) => {
-                    trace!("No tx id received");
+            Self::update_liveness_metric(&liveness_metric);
+            let tx_id = match recv.recv().await {
+                Some(tx_id) => tx_id,
+                None => {
+                    tracing::error!("Error: channel has closed");
                     break;
                 }
+            };
+
+            let logs = match indexer.fetch_logs_by_tx_hash(tx_id).await {
+                Ok(logs) => logs,
                 Err(err) => {
-                    warn!(?err, "Error receiving tx id from channel");
-                    break;
+                    warn!(?err, ?tx_id, "Error fetching logs for tx id");
+                    continue;
                 }
-            }
+            };
+
+            let logs = {
+                let store = store.lock().await;
+                Self::dedupe_and_store_logs(&domain, &store, logs, &stored_logs_metric).await
+            };
+            let num_logs = logs.len() as u64;
+            info!(
+                num_logs,
+                ?tx_id,
+                sequences = ?logs.iter().map(|(log, _)| log.sequence).collect::<Vec<_>>(),
+                pending_ids = ?recv.len(),
+                "Found log(s) for tx id"
+            );
         }
     }
 
-    #[instrument(fields(domain=self.domain().name()), skip(self, stored_logs_metric, indexed_height_metric))]
-    async fn fetch_logs_with_cursor(
-        &self,
-        cursor: &mut Box<dyn ContractSyncCursor<T>>,
-        stored_logs_metric: &GenericCounter<AtomicU64>,
-        indexed_height_metric: &GenericGauge<AtomicI64>,
+    #[allow(clippy::too_many_arguments)]
+    #[instrument(fields(domain=domain.name()), skip(indexer, store, stored_logs_metric, indexed_height_metric, liveness_metric))]
+    async fn cursor_indexer_task(
+        domain: HyperlaneDomain,
+        indexer: I,
+        store: Arc<Mutex<S>>,
+        mut cursor: Box<dyn ContractSyncCursor<T>>,
+        broadcast_sender: Option<BroadcastMpscSender<H512>>,
+        stored_logs_metric: GenericCounter<AtomicU64>,
+        indexed_height_metric: GenericGauge<AtomicI64>,
+        liveness_metric: GenericGauge<AtomicI64>,
     ) {
-        indexed_height_metric.set(cursor.latest_queried_block() as i64);
-        let (action, eta) = match cursor.next_action().await {
-            Ok((action, eta)) => (action, eta),
-            Err(err) => {
-                warn!(?err, "Error getting next action");
-                sleep(SLEEP_DURATION).await;
-                return;
-            }
-        };
-        let sleep_duration = match action {
-            // Use `loop` but always break - this allows for returning a value
-            // from the loop (the sleep duration)
-            #[allow(clippy::never_loop)]
-            CursorAction::Query(range) => loop {
-                debug!(?range, "Looking for events in index range");
-
-                let logs = match self.indexer.fetch_logs_in_range(range.clone()).await {
-                    Ok(logs) => logs,
-                    Err(err) => {
-                        warn!(?err, ?range, "Error fetching logs in range");
-                        break Some(SLEEP_DURATION);
-                    }
-                };
-
-                let logs = self.dedupe_and_store_logs(logs, stored_logs_metric).await;
-                let logs_found = logs.len() as u64;
-                info!(
-                    ?range,
-                    num_logs = logs_found,
-                    estimated_time_to_sync = fmt_sync_time(eta),
-                    sequences = ?logs.iter().map(|(log, meta)| IndexedTxIdAndSequence::new(meta.transaction_id, log.sequence)).collect::<Vec<_>>(),
-                    cursor = ?cursor,
-                    "Found log(s) in index range"
-                );
-
-                if let Some(tx) = self.broadcast_sender.as_ref() {
-                    // If multiple logs occur in the same transaction they'll have the same transaction_id.
-                    // Deduplicate their txids to avoid doing wasteful queries in txid indexer
-                    let unique_txids: HashSet<_> =
-                        logs.iter().map(|(_, meta)| meta.transaction_id).collect();
-
-                    for tx_id in unique_txids {
-                        if let Err(err) = tx.send(tx_id).await {
-                            trace!(?err, "Error sending txid to receiver");
-                        }
-                    }
+        loop {
+            Self::update_liveness_metric(&liveness_metric);
+            indexed_height_metric.set(cursor.latest_queried_block() as i64);
+
+            let (action, eta) = match cursor.next_action().await {
+                Ok((action, eta)) => (action, eta),
+                Err(err) => {
+                    warn!(?err, "Error getting next action");
+                    sleep(SLEEP_DURATION).await;
+                    continue;
+                }
+            };
+
+            let range = match action {
+                CursorAction::Sleep(duration) => {
+                    debug!(
+                        cursor = ?cursor,
+                        sleep_duration = ?duration,
+                        "Cursor can't make progress, sleeping",
+                    );
+                    sleep(duration).await;
+                    continue;
                 }
+                CursorAction::Query(range) => range,
+            };
+            debug!(?range, "Looking for events in index range");
 
-                // Update cursor
-                if let Err(err) = cursor.update(logs, range).await {
-                    warn!(?err, "Error updating cursor");
-                    break Some(SLEEP_DURATION);
-                };
-                break None;
-            },
-            CursorAction::Sleep(duration) => Some(duration),
-        };
-        if let Some(sleep_duration) = sleep_duration {
-            debug!(
+            let logs = match indexer.fetch_logs_in_range(range.clone()).await {
+                Ok(logs) => logs,
+                Err(err) => {
+                    warn!(?err, ?range, "Error fetching logs in range");
+                    sleep(SLEEP_DURATION).await;
+                    continue;
+                }
+            };
+
+            let logs = {
+                let store = store.lock().await;
+                Self::dedupe_and_store_logs(&domain, &store, logs, &stored_logs_metric).await
+            };
+            let logs_found = logs.len() as u64;
+            info!(
+                ?range,
+                num_logs = logs_found,
+                estimated_time_to_sync = fmt_sync_time(eta),
+                sequences = ?logs.iter().map(|(log, meta)| IndexedTxIdAndSequence::new(meta.transaction_id, log.sequence)).collect::<Vec<_>>(),
                 cursor = ?cursor,
-                ?sleep_duration,
-                "Cursor can't make progress, sleeping",
+                "Found log(s) in index range"
             );
-            sleep(sleep_duration).await
+
+            if let Some(tx) = broadcast_sender.as_ref() {
+                // If multiple logs occur in the same transaction they'll have the same transaction_id.
+                // Deduplicate their txids to avoid doing wasteful queries in txid indexer
+                let unique_txids: HashSet<_> =
+                    logs.iter().map(|(_, meta)| meta.transaction_id).collect();
+
+                for tx_id in unique_txids {
+                    if let Err(err) = tx.send(tx_id).await {
+                        trace!(?err, "Error sending txid to receiver");
+                    }
+                }
+            }
+
+            // Update cursor
+            if let Err(err) = cursor.update(logs, range).await {
+                warn!(?err, "Error updating cursor");
+            };
         }
     }
 
     async fn dedupe_and_store_logs(
-        &self,
+        domain: &HyperlaneDomain,
+        store: &S,
         logs: Vec<(Indexed<T>, LogMeta)>,
         stored_logs_metric: &GenericCounter<AtomicU64>,
     ) -> Vec<(Indexed<T>, LogMeta)> {
@@ -262,7 +320,7 @@ where
         let logs = Vec::from_iter(deduped_logs);
 
         // Store deliveries
-        let stored = match self.store.store_logs(&logs).await {
+        let stored = match store.store_logs(&logs).await {
             Ok(stored) => stored,
             Err(err) => {
                 warn!(?err, "Error storing logs in db");
@@ -271,7 +329,7 @@ where
         };
         if stored > 0 {
             debug!(
-                domain = self.domain.as_ref(),
+                domain = domain.name(),
                 count = stored,
                 sequences = ?logs.iter().map(|(log, _)| log.sequence).collect::<Vec<_>>(),
                 "Stored logs in db",
diff --git a/rust/main/utils/run-locally/src/main.rs b/rust/main/utils/run-locally/src/main.rs
index 1882af95ae8..7e87fe9b9c4 100644
--- a/rust/main/utils/run-locally/src/main.rs
+++ b/rust/main/utils/run-locally/src/main.rs
@@ -589,6 +589,12 @@ fn relayer_restart_invariants_met() -> eyre::Result<bool> {
     let no_metadata_message_count = *matched_logs
         .get(&line_filters)
         .ok_or_else(|| eyre::eyre!("No logs matched line filters"))?;
+
+    log!(
+        "CouldNotFetchMetadata log count found {}, expected {}",
+        no_metadata_message_count,
+        ZERO_MERKLE_INSERTION_KATHY_MESSAGES
+    );
     // These messages are never inserted into the merkle tree.
     // So these messages will never be deliverable and will always
     // be in a CouldNotFetchMetadata state.
@@ -602,10 +608,11 @@ fn relayer_restart_invariants_met() -> eyre::Result<bool> {
         );
         return Ok(false);
     }
-    assert_eq!(
-        no_metadata_message_count,
-        ZERO_MERKLE_INSERTION_KATHY_MESSAGES
-    );
+    // Technically we should be checking for strictly equals.
+    // But there are cases where the validator is behind and hasn't
+    // built metadata yet, before relayer restarts.
+    // Causing this invariant to be higher than expected
+    assert!(no_metadata_message_count >= ZERO_MERKLE_INSERTION_KATHY_MESSAGES);
     Ok(true)
 }
 


Reviews:
user: login: graphite-app[bot]
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-27T02:24:44Z
user: login: ameten
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-27T09:12:15Z
user: login: kamiyaa
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-27T14:40:52Z
user: login: daniel-savu
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-28T12:49:41Z
user: login: kamiyaa
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-28T16:18:05Z
user: login: kamiyaa
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-28T16:48:51Z
user: login: ameten
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-05-29T08:19:34Z

Review comments:
diff_hunk: @@ -256,6 +256,14 @@ pub fn scraper_termination_invariants_met(
         return Ok(false);
     }
 
+    let gas_payments_scraped = fetch_metric(
+        &scraper_metrics_port.to_string(),
+        "hyperlane_contract_sync_stored_events",
+        &hashmap! {"data_type" => "gas_payment"},
+    )?
+    .iter()
+    .for_each(|s| println!("GAS_PAYMENT: {:?}", s));
path: rust/main/utils/run-locally/src/invariants/termination_invariants.rs
user: login: graphite-app[bot]
body: There's a variable naming issue in this code. The first `gas_payments_scraped` declaration is immediately consumed by the `for_each()` call without storing any result, and then a second variable with the same name is declared right after. This creates a shadowed variable.

Consider either:
1. Removing the variable assignment in the first instance and just calling the method chain directly:
```rust
fetch_metric(
    &scraper_metrics_port.to_string(),
    "hyperlane_contract_sync_stored_events",
    &hashmap! {"data_type" => "gas_payment"},
)?
.iter()
.for_each(|s| println!("GAS_PAYMENT: {:?}", s));
```

2. Or using a different variable name for the debug output, such as `gas_payments_debug`.

This will prevent the variable shadowing and make the code's intent clearer.
```suggestion
fetch_metric(
    &scraper_metrics_port.to_string(),
    "hyperlane_contract_sync_stored_events",
    &hashmap! {"data_type" => "gas_payment"},
)?
.iter()
.for_each(|s| println!("GAS_PAYMENT: {:?}", s));
```
  

*Spotted by [Diamond](https://app.graphite.dev/diamond/?org=hyperlane-xyz&ref=ai-review-comment)*<i class='graphite__hidden'><br /><br />Is this helpful? React 👍 or 👎 to let us know.</i>
created_at: 2025-05-27T02:24:43Z
author_association: CONTRIBUTOR
diff_hunk: @@ -256,6 +256,15 @@ pub fn scraper_termination_invariants_met(
         return Ok(false);
     }
 
+    fetch_metric(
+        SCRAPER_METRICS_PORT,
+        "hyperlane_contract_sync_stored_events",
+        &hashmap! {"data_type" => "gas_payment"},
+    )?
+    .iter()
+    .map(|s: &String| s.as_str())
+    .for_each(|s| println!("GAS_PAYMENT: {:?}", s));
path: rust/main/utils/run-locally/src/invariants/termination_invariants.rs
user: login: ameten
body: It looks unrelated to this change. I wonder why we need this? Does it help with debugging of e2e tests?
created_at: 2025-05-27T09:12:15Z
author_association: CONTRIBUTOR
diff_hunk: @@ -256,6 +256,15 @@ pub fn scraper_termination_invariants_met(
         return Ok(false);
     }
 
+    fetch_metric(
+        SCRAPER_METRICS_PORT,
+        "hyperlane_contract_sync_stored_events",
+        &hashmap! {"data_type" => "gas_payment"},
+    )?
+    .iter()
+    .map(|s: &String| s.as_str())
+    .for_each(|s| println!("GAS_PAYMENT: {:?}", s));
path: rust/main/utils/run-locally/src/invariants/termination_invariants.rs
user: login: kamiyaa
body: Yea, I was testing out some stuff. Because it seemed to be passing locally, but not on CI
created_at: 2025-05-27T14:40:52Z
author_association: CONTRIBUTOR
diff_hunk: @@ -589,23 +589,25 @@ fn relayer_restart_invariants_met() -> eyre::Result<bool> {
     let no_metadata_message_count = *matched_logs
         .get(&line_filters)
         .ok_or_else(|| eyre::eyre!("No logs matched line filters"))?;
+
+    log!(
+        "CouldNotFetchMetadata log count found {}, expected {}",
+        no_metadata_message_count,
+        ZERO_MERKLE_INSERTION_KATHY_MESSAGES
+    );
     // These messages are never inserted into the merkle tree.
     // So these messages will never be deliverable and will always
     // be in a CouldNotFetchMetadata state.
     // When the relayer restarts, these messages' statuses should be
     // retrieved from the database with CouldNotFetchMetadata status.
-    if no_metadata_message_count < ZERO_MERKLE_INSERTION_KATHY_MESSAGES {
+    if no_metadata_message_count != ZERO_MERKLE_INSERTION_KATHY_MESSAGES {
path: rust/main/utils/run-locally/src/main.rs
user: login: daniel-savu
body: this invariant is intentional here. If this invariant fails there's probably a bug in the PR, or an edge case that is not well understood. Are we now indexing the same events twice because of splitting the indexing logic?
created_at: 2025-05-28T10:36:20Z
author_association: CONTRIBUTOR
diff_hunk: @@ -589,23 +589,25 @@ fn relayer_restart_invariants_met() -> eyre::Result<bool> {
     let no_metadata_message_count = *matched_logs
         .get(&line_filters)
         .ok_or_else(|| eyre::eyre!("No logs matched line filters"))?;
+
+    log!(
+        "CouldNotFetchMetadata log count found {}, expected {}",
+        no_metadata_message_count,
+        ZERO_MERKLE_INSERTION_KATHY_MESSAGES
+    );
     // These messages are never inserted into the merkle tree.
     // So these messages will never be deliverable and will always
     // be in a CouldNotFetchMetadata state.
     // When the relayer restarts, these messages' statuses should be
     // retrieved from the database with CouldNotFetchMetadata status.
-    if no_metadata_message_count < ZERO_MERKLE_INSERTION_KATHY_MESSAGES {
+    if no_metadata_message_count != ZERO_MERKLE_INSERTION_KATHY_MESSAGES {
         log!(
             "No metadata message count is {}, expected {}",
             no_metadata_message_count,
             ZERO_MERKLE_INSERTION_KATHY_MESSAGES
         );
         return Ok(false);
     }
-    assert_eq!(
-        no_metadata_message_count,
-        ZERO_MERKLE_INSERTION_KATHY_MESSAGES
-    );
path: rust/main/utils/run-locally/src/main.rs
user: login: daniel-savu
body: why remove this invariant?
created_at: 2025-05-28T10:36:59Z
author_association: CONTRIBUTOR
diff_hunk: @@ -245,8 +246,8 @@ impl BaseAgent for Relayer {
                         dbs.iter()
                             .map(|(d, db)| (d.clone(), Arc::new(db.clone())))
                             .collect(),
+                        ADVANCED_LOG_META,
                         false,
-                        settings.tx_id_indexing_enabled,
path: rust/main/agents/relayer/src/relayer.rs
user: login: daniel-savu
body: good spot - I assume we were creating broadcast channels in the IGP and merkle insertion indexers, which are supposed to consume rather than produce txids, ending up in needless memory allocations for the mpsc channel? Or is the motivation for setting `broadcast_sender_enabled` to `false` different. This should save us quite a bit of memory, curious to see the before and after
created_at: 2025-05-28T10:43:54Z
author_association: CONTRIBUTOR
diff_hunk: @@ -105,31 +107,72 @@ where
             .metrics
             .stored_events
             .with_label_values(&[label, chain_name]);
-        let liveness_metric = self
-            .metrics
-            .liveness_metrics
-            .with_label_values(&[label, chain_name]);
 
-        loop {
-            Self::update_liveness_metric(&liveness_metric);
-            if let Some(rx) = opts.tx_id_receiver.as_mut() {
-                self.fetch_logs_from_receiver(rx, &stored_logs_metric).await;
-            }
-            if let Some(cursor) = opts.cursor.as_mut() {
-                self.fetch_logs_with_cursor(cursor, &stored_logs_metric, &indexed_height_metric)
+        let shared_store = Arc::new(Mutex::new(self.store.clone()));
+
+        // transaction id task for fetching events via transaction id
+        let tx_id_task = match opts.tx_id_receiver {
+            Some(rx) => {
+                let liveness_metric = self.metrics.liveness_metrics.with_label_values(&[
+                    label,
+                    chain_name,
+                    "tx_id_task",
path: rust/main/hyperlane-base/src/contract_sync/mod.rs
user: login: daniel-savu
body: nice addition!
created_at: 2025-05-28T11:08:57Z
author_association: CONTRIBUTOR
diff_hunk: @@ -105,31 +107,72 @@ where
             .metrics
             .stored_events
             .with_label_values(&[label, chain_name]);
-        let liveness_metric = self
-            .metrics
-            .liveness_metrics
-            .with_label_values(&[label, chain_name]);
 
-        loop {
-            Self::update_liveness_metric(&liveness_metric);
-            if let Some(rx) = opts.tx_id_receiver.as_mut() {
-                self.fetch_logs_from_receiver(rx, &stored_logs_metric).await;
-            }
-            if let Some(cursor) = opts.cursor.as_mut() {
-                self.fetch_logs_with_cursor(cursor, &stored_logs_metric, &indexed_height_metric)
+        let shared_store = Arc::new(Mutex::new(self.store.clone()));
path: rust/main/hyperlane-base/src/contract_sync/mod.rs
user: login: daniel-savu
body: a comment on why this is needed would be useful
- Were you getting an error without this?
- How come it was previously not needed?

I think both Postgres and RocksDB have mechanisms for keeping reads / writes atomic. Previously exclusive DB access between the txid indexer and the cursor indexer would be guaranteed by the logic, but I'd assume now writes would be synchronized by the DB (and block when needed)
created_at: 2025-05-28T12:10:21Z
author_association: CONTRIBUTOR
diff_hunk: @@ -141,128 +184,141 @@ where
         );
     }
 
-    #[instrument(fields(domain=self.domain().name()), skip(self, recv, stored_logs_metric))]
-    async fn fetch_logs_from_receiver(
-        &self,
-        recv: &mut MpscReceiver<H512>,
-        stored_logs_metric: &GenericCounter<AtomicU64>,
+    #[instrument(fields(domain=domain.name()), skip(indexer, store, recv, stored_logs_metric, liveness_metric))]
+    async fn tx_id_receiver_task(
+        domain: HyperlaneDomain,
+        indexer: I,
+        store: Arc<Mutex<S>>,
+        mut recv: MpscReceiver<H512>,
+        stored_logs_metric: GenericCounter<AtomicU64>,
+        liveness_metric: GenericGauge<AtomicI64>,
     ) {
         loop {
-            match recv.try_recv() {
-                Ok(tx_id) => {
-                    let logs = match self.indexer.fetch_logs_by_tx_hash(tx_id).await {
-                        Ok(logs) => logs,
-                        Err(err) => {
-                            warn!(?err, ?tx_id, "Error fetching logs for tx id");
-                            continue;
-                        }
-                    };
-                    let logs = self.dedupe_and_store_logs(logs, stored_logs_metric).await;
-                    let num_logs = logs.len() as u64;
-                    info!(
-                        num_logs,
-                        ?tx_id,
-                        sequences = ?logs.iter().map(|(log, _)| log.sequence).collect::<Vec<_>>(),
-                        pending_ids = ?recv.len(),
-                        "Found log(s) for tx id"
-                    );
-                }
-                Err(TryRecvError::Empty) => {
-                    trace!("No tx id received");
+            Self::update_liveness_metric(&liveness_metric);
+            let tx_id = match recv.recv().await {
+                Some(tx_id) => tx_id,
+                None => {
+                    warn!("Error: channel has closed");
path: rust/main/hyperlane-base/src/contract_sync/mod.rs
user: login: daniel-savu
body: worth having this log be at `error` level
created_at: 2025-05-28T12:15:00Z
author_association: CONTRIBUTOR
diff_hunk: @@ -141,128 +184,141 @@ where
         );
     }
 
-    #[instrument(fields(domain=self.domain().name()), skip(self, recv, stored_logs_metric))]
-    async fn fetch_logs_from_receiver(
-        &self,
-        recv: &mut MpscReceiver<H512>,
-        stored_logs_metric: &GenericCounter<AtomicU64>,
+    #[instrument(fields(domain=domain.name()), skip(indexer, store, recv, stored_logs_metric, liveness_metric))]
+    async fn tx_id_receiver_task(
+        domain: HyperlaneDomain,
+        indexer: I,
+        store: Arc<Mutex<S>>,
+        mut recv: MpscReceiver<H512>,
+        stored_logs_metric: GenericCounter<AtomicU64>,
+        liveness_metric: GenericGauge<AtomicI64>,
     ) {
         loop {
-            match recv.try_recv() {
-                Ok(tx_id) => {
-                    let logs = match self.indexer.fetch_logs_by_tx_hash(tx_id).await {
-                        Ok(logs) => logs,
-                        Err(err) => {
-                            warn!(?err, ?tx_id, "Error fetching logs for tx id");
-                            continue;
-                        }
-                    };
-                    let logs = self.dedupe_and_store_logs(logs, stored_logs_metric).await;
-                    let num_logs = logs.len() as u64;
-                    info!(
-                        num_logs,
-                        ?tx_id,
-                        sequences = ?logs.iter().map(|(log, _)| log.sequence).collect::<Vec<_>>(),
-                        pending_ids = ?recv.len(),
-                        "Found log(s) for tx id"
-                    );
-                }
-                Err(TryRecvError::Empty) => {
-                    trace!("No tx id received");
+            Self::update_liveness_metric(&liveness_metric);
+            let tx_id = match recv.recv().await {
+                Some(tx_id) => tx_id,
+                None => {
+                    warn!("Error: channel has closed");
                     break;
                 }
+            };
+
+            let logs = match indexer.fetch_logs_by_tx_hash(tx_id).await {
+                Ok(logs) => logs,
                 Err(err) => {
-                    warn!(?err, "Error receiving tx id from channel");
-                    break;
+                    warn!(?err, ?tx_id, "Error fetching logs for tx id");
+                    continue;
                 }
-            }
+            };
+
+            let logs = {
+                let store = store.lock().await;
+                Self::dedupe_and_store_logs(&domain, &store, logs, &stored_logs_metric).await
+            };
+            let num_logs = logs.len() as u64;
+            info!(
+                num_logs,
+                ?tx_id,
+                sequences = ?logs.iter().map(|(log, _)| log.sequence).collect::<Vec<_>>(),
+                pending_ids = ?recv.len(),
+                "Found log(s) for tx id"
+            );
         }
     }
 
-    #[instrument(fields(domain=self.domain().name()), skip(self, stored_logs_metric, indexed_height_metric))]
-    async fn fetch_logs_with_cursor(
-        &self,
-        cursor: &mut Box<dyn ContractSyncCursor<T>>,
-        stored_logs_metric: &GenericCounter<AtomicU64>,
-        indexed_height_metric: &GenericGauge<AtomicI64>,
+    #[allow(clippy::too_many_arguments)]
+    #[instrument(fields(domain=domain.name()), skip(indexer, store, stored_logs_metric, indexed_height_metric, liveness_metric))]
+    async fn cursor_receiver_task(
path: rust/main/hyperlane-base/src/contract_sync/mod.rs
user: login: daniel-savu
body: nit: since the cursor doesn't receive anything, a better name would be `cursor_indexer_task`
created_at: 2025-05-28T12:18:05Z
author_association: CONTRIBUTOR
diff_hunk: @@ -141,128 +184,141 @@ where
         );
     }
 
-    #[instrument(fields(domain=self.domain().name()), skip(self, recv, stored_logs_metric))]
-    async fn fetch_logs_from_receiver(
-        &self,
-        recv: &mut MpscReceiver<H512>,
-        stored_logs_metric: &GenericCounter<AtomicU64>,
+    #[instrument(fields(domain=domain.name()), skip(indexer, store, recv, stored_logs_metric, liveness_metric))]
+    async fn tx_id_receiver_task(
path: rust/main/hyperlane-base/src/contract_sync/mod.rs
user: login: daniel-savu
body: nit: rename to `tx_id_indexer_task`
created_at: 2025-05-28T12:18:21Z
author_association: CONTRIBUTOR
diff_hunk: @@ -105,31 +107,72 @@ where
             .metrics
             .stored_events
             .with_label_values(&[label, chain_name]);
-        let liveness_metric = self
-            .metrics
-            .liveness_metrics
-            .with_label_values(&[label, chain_name]);
 
-        loop {
-            Self::update_liveness_metric(&liveness_metric);
-            if let Some(rx) = opts.tx_id_receiver.as_mut() {
-                self.fetch_logs_from_receiver(rx, &stored_logs_metric).await;
-            }
-            if let Some(cursor) = opts.cursor.as_mut() {
-                self.fetch_logs_with_cursor(cursor, &stored_logs_metric, &indexed_height_metric)
+        let shared_store = Arc::new(Mutex::new(self.store.clone()));
+
+        // transaction id task for fetching events via transaction id
+        let tx_id_task = match opts.tx_id_receiver {
+            Some(rx) => {
+                let liveness_metric = self.metrics.liveness_metrics.with_label_values(&[
+                    label,
+                    chain_name,
+                    "tx_id_task",
+                ]);
+                let domain_clone = self.domain.clone();
+                let indexer_clone = self.indexer.clone();
+                let store_clone = shared_store.clone();
+                let stored_logs_metric = stored_logs_metric.clone();
+                tokio::task::spawn(async move {
+                    Self::tx_id_receiver_task(
+                        domain_clone,
+                        indexer_clone,
+                        store_clone,
+                        rx,
+                        stored_logs_metric,
+                        liveness_metric,
+                    )
                     .await;
+                })
             }
+            None => tokio::task::spawn(async {}),
+        };
 
-            // Added so that we confuse compiler that it is an infinite loop
-            if false {
-                break;
+        // cursor task for fetching events via range querying
+        let cursor_task = match opts.cursor {
+            Some(cursor) => {
+                let liveness_metric = self.metrics.liveness_metrics.with_label_values(&[
+                    label,
+                    chain_name,
+                    "cursor_task",
+                ]);
+                let domain_clone = self.domain.clone();
+                let indexer_clone = self.indexer.clone();
+                let store_clone = shared_store.clone();
+                let broadcast_sender = self.broadcast_sender.clone();
+
+                let stored_logs_metric = stored_logs_metric.clone();
+
+                tokio::task::spawn(async {
+                    Self::cursor_receiver_task(
+                        domain_clone,
+                        indexer_clone,
+                        store_clone,
+                        cursor,
+                        broadcast_sender,
+                        stored_logs_metric,
+                        indexed_height_metric,
+                        liveness_metric,
+                    )
+                    .await;
+                })
             }
-        }
+            None => tokio::task::spawn(async {}),
+        };
 
-        // Although the above loop should never end (unless by panicking),
-        // we put log here to make sure that we see when this method returns normally.
-        // Hopefully, compiler will not optimise this code out.
-        info!(chain = chain_name, label, "contract sync loop exit");
+        let res = tokio::join!(tx_id_task, cursor_task);
+
+        // we should never reach this because the 2 tasks should never end
+        warn!(chain = chain_name, label, ?res, "contract sync loop exit");
path: rust/main/hyperlane-base/src/contract_sync/mod.rs
user: login: daniel-savu
body: good point making this a `warn`, i'd even make it an `error` actually
created_at: 2025-05-28T12:28:35Z
author_association: CONTRIBUTOR
diff_hunk: @@ -589,23 +589,25 @@ fn relayer_restart_invariants_met() -> eyre::Result<bool> {
     let no_metadata_message_count = *matched_logs
         .get(&line_filters)
         .ok_or_else(|| eyre::eyre!("No logs matched line filters"))?;
path: rust/main/utils/run-locally/src/main.rs
user: login: daniel-savu
body: related to testing in general rather than to this code line: did you notice a drop in message delivery latency due to this change? I would expect so since we're no longer blocked by the rate limits, would be good to check by looking at the timestamps of `Operation submitted` logs and seeing if they occur earlier in E2E run now
created_at: 2025-05-28T12:43:18Z
author_association: CONTRIBUTOR
diff_hunk: @@ -589,23 +589,25 @@ fn relayer_restart_invariants_met() -> eyre::Result<bool> {
     let no_metadata_message_count = *matched_logs
         .get(&line_filters)
         .ok_or_else(|| eyre::eyre!("No logs matched line filters"))?;
path: rust/main/utils/run-locally/src/main.rs
user: login: kamiyaa
body: Yeah, theres about a 30s improvement from loading settings to first Operation submitted log
```
// before changes
Loading Settings     : 2025-05-28T13:35:05.823745793Z
Operation submitted  : 2025-05-28T13:36:19.963864Z
                       2025-05-28T13:36:57.767831Z

// after changes
Loading Settings     : 2025-05-28T13:23:30.300770674Z
Operation submitted  : 2025-05-28T13:24:01.310479Z
                       2025-05-28T13:24:03.341528Z
                       2025-05-28T13:24:05.370521Z
```
created_at: 2025-05-28T16:18:05Z
author_association: CONTRIBUTOR
diff_hunk: @@ -245,8 +246,8 @@ impl BaseAgent for Relayer {
                         dbs.iter()
                             .map(|(d, db)| (d.clone(), Arc::new(db.clone())))
                             .collect(),
+                        ADVANCED_LOG_META,
                         false,
-                        settings.tx_id_indexing_enabled,
path: rust/main/agents/relayer/src/relayer.rs
user: login: kamiyaa
body: > I assume we were creating broadcast channels in the IGP and merkle insertion indexers, which are supposed to consume rather than produce txids, ending up in needless memory allocations for the mpsc channel?

Yeah exactly :+1:
created_at: 2025-05-28T16:48:51Z
author_association: CONTRIBUTOR

PR comments:
user: login: changeset-bot[bot]
created_at: 2025-05-26T17:25:31Z
body: ###  ⚠️  No Changeset found

Latest commit: 9c998edb5f3f343653953873cd24407df3aa7c97

Merging this PR will not cause a version bump for any packages. If these changes should not result in a new version, you're good to go. **If these changes should result in a version bump, you need to add a changeset.**

<details><summary>This PR includes no changesets</summary>

  When changesets are added to this PR, you'll see the packages that this PR includes changesets for and the associated semver types

</details>

[Click here to learn what changesets are, and how to add one](https://github.com/changesets/changesets/blob/main/docs/adding-a-changeset.md).

[Click here if you're a maintainer who wants to add a changeset to this PR](https://github.com/hyperlane-xyz/hyperlane-monorepo/new/jeff/igp-separation-2?filename=.changeset/grumpy-phones-check.md&value=---%0A%0A---%0A%0Afeat%3A%20separate%20tx%20id%20indexing%20and%20cursor%20indexing%20into%20separate%20threads%0A)


author_association: NONE
user: login: codecov[bot]
created_at: 2025-05-26T17:27:09Z
body: ## [Codecov](https://app.codecov.io/gh/hyperlane-xyz/hyperlane-monorepo/pull/6350?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=hyperlane-xyz) Report
All modified and coverable lines are covered by tests :white_check_mark:
> Project coverage is 0.00%. Comparing base [(`11b240a`)](https://app.codecov.io/gh/hyperlane-xyz/hyperlane-monorepo/commit/11b240a29a24060ed34e69f103ee84ba5ea96b04?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=hyperlane-xyz) to head [(`9c998ed`)](https://app.codecov.io/gh/hyperlane-xyz/hyperlane-monorepo/commit/9c998edb5f3f343653953873cd24407df3aa7c97?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=hyperlane-xyz).
> Report is 9 commits behind head on main.

<details><summary>Additional details and impacted files</summary>


```diff
@@          Coverage Diff          @@
##            main   #6350   +/-   ##
=====================================
  Coverage   0.00%   0.00%           
=====================================
  Files          1       1           
  Lines         14      14           
=====================================
  Misses        14      14           
```

| [Components](https://app.codecov.io/gh/hyperlane-xyz/hyperlane-monorepo/pull/6350/components?src=pr&el=components&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=hyperlane-xyz) | Coverage Δ | |
|---|---|---|
| [core](https://app.codecov.io/gh/hyperlane-xyz/hyperlane-monorepo/pull/6350/components?src=pr&el=component&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=hyperlane-xyz) | `∅ <ø> (∅)` | |
| [hooks](https://app.codecov.io/gh/hyperlane-xyz/hyperlane-monorepo/pull/6350/components?src=pr&el=component&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=hyperlane-xyz) | `∅ <ø> (∅)` | |
| [isms](https://app.codecov.io/gh/hyperlane-xyz/hyperlane-monorepo/pull/6350/components?src=pr&el=component&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=hyperlane-xyz) | `∅ <ø> (∅)` | |
| [token](https://app.codecov.io/gh/hyperlane-xyz/hyperlane-monorepo/pull/6350/components?src=pr&el=component&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=hyperlane-xyz) | `∅ <ø> (∅)` | |
| [middlewares](https://app.codecov.io/gh/hyperlane-xyz/hyperlane-monorepo/pull/6350/components?src=pr&el=component&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=hyperlane-xyz) | `∅ <ø> (∅)` | |

</details>
<details><summary> :rocket: New features to boost your workflow: </summary>

- :snowflake: [Test Analytics](https://docs.codecov.com/docs/test-analytics): Detect flaky tests, report on failures, and find test suite problems.
- :package: [JS Bundle Analysis](https://docs.codecov.com/docs/javascript-bundle-analysis): Save yourself from yourself by tracking and limiting bundle sizes in JS merges.
</details>
author_association: NONE
user: login: ameten
created_at: 2025-05-27T09:13:48Z
body: It is much clearer what is happening in this change now. Thank you, @kamiyaa, for coming up with the solution!
author_association: CONTRIBUTOR
user: login: kamiyaa
created_at: 2025-05-28T16:29:37Z
body: ## Issue

Debugged with @tkporter to figure out why we were running into this error:
```python
2025-05-28T15:23:38.3726733Z <E2E> Checking message statuses were retrieved from logs...
2025-05-28T15:23:38.3775725Z thread 'main' panicked at utils/run-locally/src/main.rs:611:5:
2025-05-28T15:23:38.3776067Z assertion `left == right` failed
2025-05-28T15:23:38.3776279Z   left: 11
2025-05-28T15:23:38.3776435Z  right: 10
2025-05-28T15:23:38.3776595Z stack backtrace:
2025-05-28T15:23:38.3776901Z <E2E> CouldNotFetchMetadata log count found 11, expected 10
```

### Cause
Our leading theory is that E2E is running too fast now for validators to keep up.
This causes some messages to be undeliverable while the relayer is running due to `CouldNotFetchMetadata`, but become deliverable after the relayer has restarted.

The transaction in question: `0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730`

Timestamp of when relayer restarted:
```python
~$ cat job-logs.txt | grep -i restart
2025-05-28T15:18:53.4625070Z No services need to be restarted.
2025-05-28T15:18:53.4625334Z No containers need to be restarted.
2025-05-28T15:23:13.3715154Z <E2E> Restarting relayer...
2025-05-28T15:23:13.3717152Z <E2E> Restarted relayer...
```

Timestamp of when validator inserted metadata:
```python
~$ ID=0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730
~$ cat job-logs.txt | grep "$ID" | grep VL
2025-05-28T15:23:15.7033879Z <VL1> 2025-05-28T15:23:15.703000Z  INFO TipCheckpointSubmitter: validator::submit: Ingested leaf to in-memory merkle tree index=13 root=0x356862309e0c09e3b3058c9d2d0eeb84c552b417e4d4903616dc3ef395ff14d1 message_id=0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730 queue_length=0
```
So merkle tree was inserted 2s after relayer restarted. Hence we had 11 CouldNotFetchMetadata logs instead of 10

### Solution

Relax the E2E invariant to be >= instead of strictly equal

#### Additional notes

Subsequent logs of `0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730` being processed:
```python
2025-05-28T15:23:19.6743661Z <SCR> 2025-05-28T15:23:19.673829Z DEBUG cursor_indexer_task: scraper::store::payments: storable payments domain=9913371 interchain_gas_paymaster_address=0x0000000000000000000000005eb3bc0a489c5a8288765d2336659ebca68fcd00 storable=[StorablePayment { payment: InterchainGasPayment { message_id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, destination: 9913372, payment: 1050000000000000, gas_amount: 1050000 }, sequence: None, meta: LogMeta { address: 0x0000000000000000000000005eb3bc0a489c5a8288765d2336659ebca68fcd00, block_number: 408, block_hash: 0x7f93a99fc6b9e3a36c7cb0750b484b30159690da16a7b0f32cfe71585497a40e, transaction_id: 0x0000000000000000000000000000000000000000000000000000000000000000aecf177bdd8e33c8fbd9572024fdab96a32291d4db682acc9d6a18474720d196, transaction_index: 0, log_index: 3 }, txn_id: 43 }] cursor=RateLimitedContractSyncCursor { tip: 138, last_tip_update: Instant { tv_sec: 297, tv_nsec: 957401903 }, sync_state: SyncState { chunk_size: 1, start_block: 30, next_block: 30, direction: Forward }, domain: HyperlaneDomain(test1 (9913371)) } broadcast_sender=None domain="test1"
2025-05-28T15:23:29.5185769Z <RLY> 2025-05-28T15:23:29.517249Z DEBUG MessageProcessor:main_loop: relayer::msg::processor: Found processable message hyp_message=HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 } iterator=DirectionalNonceIterator { nonce: Some(13), direction: High, domain: "test1" } origin=test1 domain=test1
2025-05-28T15:23:29.5207091Z <RLY> 2025-05-28T15:23:29.519157Z DEBUG MessageProcessor:main_loop: relayer::msg::processor: Processor working on message msg=HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 } cursor=ForwardBackwardIterator { low_nonce_iter: DirectionalNonceIterator { nonce: Some(12), direction: Low, domain: "test1" }, high_nonce_iter: DirectionalNonceIterator { nonce: Some(14), direction: High, domain: "test1" }, _domain: "test1" } origin=test1 domain=test1
2025-05-28T15:23:29.5238171Z <RLY> 2025-05-28T15:23:29.520425Z DEBUG MessageProcessor:main_loop: relayer::msg::processor: Sending message to submitter msg=HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 } origin=test1 domain=test1
2025-05-28T15:23:29.5249384Z <RLY> 2025-05-28T15:23:29.523942Z DEBUG MessageProcessor:main_loop: relayer::msg::pending_message: status=Retry(CouldNotFetchMetadata) id=0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730 RETRIEVED_MESSAGE_LOG="Message status retrieved from db" origin=test1 domain=test1
2025-05-28T15:23:29.5726793Z <RLY> 2025-05-28T15:23:29.570342Z DEBUG MerkleTreeProcessor:main_loop: relayer::merkle_tree::builder: Ingesting leaf message_id=0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730 origin=test1 domain=test1
2025-05-28T15:23:29.6030677Z <RLY> 2025-05-28T15:23:29.584778Z DEBUG prepare_classic_task:pop_many: relayer::msg::op_queue: Popped OpQueue operations queue_label=prepare_queue operations=[PendingMessage { num_retries: 1, since_last_attempt_s: 0, next_attempt_after_s: 4, message: HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }, PendingMessage { num_retries: 4, since_last_attempt_s: 0, next_attempt_after_s: 59, message: HyperlaneMessage { id: 0x4b33454318f3e67fe733e7fe6c7a877eeca6a5e9fbb224c51eb183ad1dd61383, version: 3, nonce: 1, origin: test3, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x5d42ebdbba61412295d7b0302d6f50ac449ddb4f, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }, PendingMessage { num_retries: 4, since_last_attempt_s: 0, next_attempt_after_s: 59, message: HyperlaneMessage { id: 0xaf25f3da1da8917e47542818eb65fe6acefd3e2c8d74ecedcc4075bbe2669e7c, version: 3, nonce: 10, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0xd9fec8238711935d6c8d79bef2b9546ef23fc046, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }, PendingMessage { num_retries: 4, since_last_attempt_s: 0, next_attempt_after_s: 59, message: HyperlaneMessage { id: 0x300188dea2c304fa9b7d96e9f8983fd6cd1357b58853dd3124c5b8157299b345, version: 3, nonce: 9, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0xd9fec8238711935d6c8d79bef2b9546ef23fc046, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }, PendingMessage { num_retries: 4, since_last_attempt_s: 0, next_attempt_after_s: 59, message: HyperlaneMessage { id: 0x4bc6305e53882847f4a9993dbffc708cf20bb356973bda98ba4ddfbb1460733f, version: 3, nonce: 3, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x5d42ebdbba61412295d7b0302d6f50ac449ddb4f, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }] domain=test2 limit=5 queue_label=prepare_queue
2025-05-28T15:23:30.0917866Z <RLY> 2025-05-28T15:23:30.090376Z DEBUG prepare_classic_task:pop_many: relayer::msg::op_queue: Popped OpQueue operations queue_label=prepare_queue operations=[PendingMessage { num_retries: 1, since_last_attempt_s: 0, next_attempt_after_s: 4, message: HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }, PendingMessage { num_retries: 4, since_last_attempt_s: 0, next_attempt_after_s: 59, message: HyperlaneMessage { id: 0x4b33454318f3e67fe733e7fe6c7a877eeca6a5e9fbb224c51eb183ad1dd61383, version: 3, nonce: 1, origin: test3, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x5d42ebdbba61412295d7b0302d6f50ac449ddb4f, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }, PendingMessage { num_retries: 4, since_last_attempt_s: 0, next_attempt_after_s: 59, message: HyperlaneMessage { id: 0xaf25f3da1da8917e47542818eb65fe6acefd3e2c8d74ecedcc4075bbe2669e7c, version: 3, nonce: 10, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0xd9fec8238711935d6c8d79bef2b9546ef23fc046, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }, PendingMessage { num_retries: 4, since_last_attempt_s: 0, next_attempt_after_s: 59, message: HyperlaneMessage { id: 0x300188dea2c304fa9b7d96e9f8983fd6cd1357b58853dd3124c5b8157299b345, version: 3, nonce: 9, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0xd9fec8238711935d6c8d79bef2b9546ef23fc046, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }, PendingMessage { num_retries: 4, since_last_attempt_s: 0, next_attempt_after_s: 59, message: HyperlaneMessage { id: 0x4bc6305e53882847f4a9993dbffc708cf20bb356973bda98ba4ddfbb1460733f, version: 3, nonce: 3, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x5d42ebdbba61412295d7b0302d6f50ac449ddb4f, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }] domain=test2 limit=5 queue_label=prepare_queue
2025-05-28T15:23:30.5944422Z <RLY> 2025-05-28T15:23:30.593002Z DEBUG prepare_classic_task:pop_many: relayer::msg::op_queue: Popped OpQueue operations queue_label=prepare_queue operations=[PendingMessage { num_retries: 1, since_last_attempt_s: 1, next_attempt_after_s: 3, message: HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }, PendingMessage { num_retries: 4, since_last_attempt_s: 1, next_attempt_after_s: 58, message: HyperlaneMessage { id: 0x4b33454318f3e67fe733e7fe6c7a877eeca6a5e9fbb224c51eb183ad1dd61383, version: 3, nonce: 1, origin: test3, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x5d42ebdbba61412295d7b0302d6f50ac449ddb4f, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }, PendingMessage { num_retries: 4, since_last_attempt_s: 1, next_attempt_after_s: 58, message: HyperlaneMessage { id: 0xaf25f3da1da8917e47542818eb65fe6acefd3e2c8d74ecedcc4075bbe2669e7c, version: 3, nonce: 10, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0xd9fec8238711935d6c8d79bef2b9546ef23fc046, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }, PendingMessage { num_retries: 4, since_last_attempt_s: 1, next_attempt_after_s: 58, message: HyperlaneMessage { id: 0x300188dea2c304fa9b7d96e9f8983fd6cd1357b58853dd3124c5b8157299b345, version: 3, nonce: 9, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0xd9fec8238711935d6c8d79bef2b9546ef23fc046, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }, PendingMessage { num_retries: 4, since_last_attempt_s: 1, next_attempt_after_s: 58, message: HyperlaneMessage { id: 0x4bc6305e53882847f4a9993dbffc708cf20bb356973bda98ba4ddfbb1460733f, version: 3, nonce: 3, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x5d42ebdbba61412295d7b0302d6f50ac449ddb4f, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }] domain=test2 limit=5 queue_label=prepare_queue
2025-05-28T15:23:31.0971337Z <RLY> 2025-05-28T15:23:31.095692Z DEBUG prepare_classic_task:pop_many: relayer::msg::op_queue: Popped OpQueue operations queue_label=prepare_queue operations=[PendingMessage { num_retries: 1, since_last_attempt_s: 1, next_attempt_after_s: 3, message: HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }, PendingMessage { num_retries: 4, since_last_attempt_s: 1, next_attempt_after_s: 58, message: HyperlaneMessage { id: 0x4b33454318f3e67fe733e7fe6c7a877eeca6a5e9fbb224c51eb183ad1dd61383, version: 3, nonce: 1, origin: test3, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x5d42ebdbba61412295d7b0302d6f50ac449ddb4f, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }, PendingMessage { num_retries: 4, since_last_attempt_s: 1, next_attempt_after_s: 58, message: HyperlaneMessage { id: 0xaf25f3da1da8917e47542818eb65fe6acefd3e2c8d74ecedcc4075bbe2669e7c, version: 3, nonce: 10, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0xd9fec8238711935d6c8d79bef2b9546ef23fc046, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }, PendingMessage { num_retries: 4, since_last_attempt_s: 1, next_attempt_after_s: 58, message: HyperlaneMessage { id: 0x300188dea2c304fa9b7d96e9f8983fd6cd1357b58853dd3124c5b8157299b345, version: 3, nonce: 9, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0xd9fec8238711935d6c8d79bef2b9546ef23fc046, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }, PendingMessage { num_retries: 4, since_last_attempt_s: 1, next_attempt_after_s: 58, message: HyperlaneMessage { id: 0x4bc6305e53882847f4a9993dbffc708cf20bb356973bda98ba4ddfbb1460733f, version: 3, nonce: 3, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x5d42ebdbba61412295d7b0302d6f50ac449ddb4f, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }] domain=test2 limit=5 queue_label=prepare_queue
2025-05-28T15:23:31.5986006Z <RLY> 2025-05-28T15:23:31.597191Z DEBUG prepare_classic_task:pop_many: relayer::msg::op_queue: Popped OpQueue operations queue_label=prepare_queue operations=[PendingMessage { num_retries: 1, since_last_attempt_s: 2, next_attempt_after_s: 2, message: HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }, PendingMessage { num_retries: 4, since_last_attempt_s: 2, next_attempt_after_s: 57, message: HyperlaneMessage { id: 0x4b33454318f3e67fe733e7fe6c7a877eeca6a5e9fbb224c51eb183ad1dd61383, version: 3, nonce: 1, origin: test3, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x5d42ebdbba61412295d7b0302d6f50ac449ddb4f, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }, PendingMessage { num_retries: 4, since_last_attempt_s: 2, next_attempt_after_s: 57, message: HyperlaneMessage { id: 0xaf25f3da1da8917e47542818eb65fe6acefd3e2c8d74ecedcc4075bbe2669e7c, version: 3, nonce: 10, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0xd9fec8238711935d6c8d79bef2b9546ef23fc046, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }, PendingMessage { num_retries: 4, since_last_attempt_s: 2, next_attempt_after_s: 57, message: HyperlaneMessage { id: 0x300188dea2c304fa9b7d96e9f8983fd6cd1357b58853dd3124c5b8157299b345, version: 3, nonce: 9, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0xd9fec8238711935d6c8d79bef2b9546ef23fc046, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }, PendingMessage { num_retries: 4, since_last_attempt_s: 2, next_attempt_after_s: 57, message: HyperlaneMessage { id: 0x4bc6305e53882847f4a9993dbffc708cf20bb356973bda98ba4ddfbb1460733f, version: 3, nonce: 3, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x5d42ebdbba61412295d7b0302d6f50ac449ddb4f, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }] domain=test2 limit=5 queue_label=prepare_queue
2025-05-28T15:23:32.1008032Z <RLY> 2025-05-28T15:23:32.099393Z DEBUG prepare_classic_task:pop_many: relayer::msg::op_queue: Popped OpQueue operations queue_label=prepare_queue operations=[PendingMessage { num_retries: 1, since_last_attempt_s: 2, next_attempt_after_s: 2, message: HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }, PendingMessage { num_retries: 4, since_last_attempt_s: 2, next_attempt_after_s: 57, message: HyperlaneMessage { id: 0x4b33454318f3e67fe733e7fe6c7a877eeca6a5e9fbb224c51eb183ad1dd61383, version: 3, nonce: 1, origin: test3, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x5d42ebdbba61412295d7b0302d6f50ac449ddb4f, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }, PendingMessage { num_retries: 4, since_last_attempt_s: 2, next_attempt_after_s: 57, message: HyperlaneMessage { id: 0xaf25f3da1da8917e47542818eb65fe6acefd3e2c8d74ecedcc4075bbe2669e7c, version: 3, nonce: 10, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0xd9fec8238711935d6c8d79bef2b9546ef23fc046, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }, PendingMessage { num_retries: 4, since_last_attempt_s: 2, next_attempt_after_s: 57, message: HyperlaneMessage { id: 0x300188dea2c304fa9b7d96e9f8983fd6cd1357b58853dd3124c5b8157299b345, version: 3, nonce: 9, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0xd9fec8238711935d6c8d79bef2b9546ef23fc046, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }, PendingMessage { num_retries: 4, since_last_attempt_s: 2, next_attempt_after_s: 57, message: HyperlaneMessage { id: 0x4bc6305e53882847f4a9993dbffc708cf20bb356973bda98ba4ddfbb1460733f, version: 3, nonce: 3, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x5d42ebdbba61412295d7b0302d6f50ac449ddb4f, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }] domain=test2 limit=5 queue_label=prepare_queue
2025-05-28T15:23:32.6033831Z <RLY> 2025-05-28T15:23:32.601959Z DEBUG prepare_classic_task:pop_many: relayer::msg::op_queue: Popped OpQueue operations queue_label=prepare_queue operations=[PendingMessage { num_retries: 1, since_last_attempt_s: 3, next_attempt_after_s: 1, message: HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }, PendingMessage { num_retries: 4, since_last_attempt_s: 3, next_attempt_after_s: 56, message: HyperlaneMessage { id: 0x4b33454318f3e67fe733e7fe6c7a877eeca6a5e9fbb224c51eb183ad1dd61383, version: 3, nonce: 1, origin: test3, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x5d42ebdbba61412295d7b0302d6f50ac449ddb4f, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }, PendingMessage { num_retries: 4, since_last_attempt_s: 3, next_attempt_after_s: 56, message: HyperlaneMessage { id: 0xaf25f3da1da8917e47542818eb65fe6acefd3e2c8d74ecedcc4075bbe2669e7c, version: 3, nonce: 10, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0xd9fec8238711935d6c8d79bef2b9546ef23fc046, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }, PendingMessage { num_retries: 4, since_last_attempt_s: 3, next_attempt_after_s: 56, message: HyperlaneMessage { id: 0x300188dea2c304fa9b7d96e9f8983fd6cd1357b58853dd3124c5b8157299b345, version: 3, nonce: 9, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0xd9fec8238711935d6c8d79bef2b9546ef23fc046, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }, PendingMessage { num_retries: 4, since_last_attempt_s: 3, next_attempt_after_s: 56, message: HyperlaneMessage { id: 0x4bc6305e53882847f4a9993dbffc708cf20bb356973bda98ba4ddfbb1460733f, version: 3, nonce: 3, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x5d42ebdbba61412295d7b0302d6f50ac449ddb4f, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }] domain=test2 limit=5 queue_label=prepare_queue
2025-05-28T15:23:33.1061957Z <RLY> 2025-05-28T15:23:33.104778Z DEBUG prepare_classic_task:pop_many: relayer::msg::op_queue: Popped OpQueue operations queue_label=prepare_queue operations=[PendingMessage { num_retries: 1, since_last_attempt_s: 3, next_attempt_after_s: 1, message: HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }, PendingMessage { num_retries: 4, since_last_attempt_s: 3, next_attempt_after_s: 56, message: HyperlaneMessage { id: 0x4b33454318f3e67fe733e7fe6c7a877eeca6a5e9fbb224c51eb183ad1dd61383, version: 3, nonce: 1, origin: test3, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x5d42ebdbba61412295d7b0302d6f50ac449ddb4f, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }, PendingMessage { num_retries: 4, since_last_attempt_s: 3, next_attempt_after_s: 56, message: HyperlaneMessage { id: 0xaf25f3da1da8917e47542818eb65fe6acefd3e2c8d74ecedcc4075bbe2669e7c, version: 3, nonce: 10, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0xd9fec8238711935d6c8d79bef2b9546ef23fc046, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }, PendingMessage { num_retries: 4, since_last_attempt_s: 3, next_attempt_after_s: 56, message: HyperlaneMessage { id: 0x300188dea2c304fa9b7d96e9f8983fd6cd1357b58853dd3124c5b8157299b345, version: 3, nonce: 9, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0xd9fec8238711935d6c8d79bef2b9546ef23fc046, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }, PendingMessage { num_retries: 4, since_last_attempt_s: 3, next_attempt_after_s: 56, message: HyperlaneMessage { id: 0x4bc6305e53882847f4a9993dbffc708cf20bb356973bda98ba4ddfbb1460733f, version: 3, nonce: 3, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x5d42ebdbba61412295d7b0302d6f50ac449ddb4f, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }] domain=test2 limit=5 queue_label=prepare_queue
2025-05-28T15:23:33.6087509Z <RLY> 2025-05-28T15:23:33.607346Z DEBUG prepare_classic_task:pop_many: relayer::msg::op_queue: Popped OpQueue operations queue_label=prepare_queue operations=[PendingMessage { num_retries: 1, since_last_attempt_s: 4, next_attempt_after_s: 0, message: HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }, PendingMessage { num_retries: 4, since_last_attempt_s: 4, next_attempt_after_s: 55, message: HyperlaneMessage { id: 0x4b33454318f3e67fe733e7fe6c7a877eeca6a5e9fbb224c51eb183ad1dd61383, version: 3, nonce: 1, origin: test3, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x5d42ebdbba61412295d7b0302d6f50ac449ddb4f, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }, PendingMessage { num_retries: 4, since_last_attempt_s: 4, next_attempt_after_s: 55, message: HyperlaneMessage { id: 0xaf25f3da1da8917e47542818eb65fe6acefd3e2c8d74ecedcc4075bbe2669e7c, version: 3, nonce: 10, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0xd9fec8238711935d6c8d79bef2b9546ef23fc046, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }, PendingMessage { num_retries: 4, since_last_attempt_s: 4, next_attempt_after_s: 55, message: HyperlaneMessage { id: 0x300188dea2c304fa9b7d96e9f8983fd6cd1357b58853dd3124c5b8157299b345, version: 3, nonce: 9, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0xd9fec8238711935d6c8d79bef2b9546ef23fc046, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }, PendingMessage { num_retries: 4, since_last_attempt_s: 4, next_attempt_after_s: 55, message: HyperlaneMessage { id: 0x4bc6305e53882847f4a9993dbffc708cf20bb356973bda98ba4ddfbb1460733f, version: 3, nonce: 3, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x5d42ebdbba61412295d7b0302d6f50ac449ddb4f, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }] domain=test2 limit=5 queue_label=prepare_queue
2025-05-28T15:23:34.1115612Z <RLY> 2025-05-28T15:23:34.110139Z DEBUG prepare_classic_task:pop_many: relayer::msg::op_queue: Popped OpQueue operations queue_label=prepare_queue operations=[PendingMessage { num_retries: 1, since_last_attempt_s: 4, next_attempt_after_s: 0, message: HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }, PendingMessage { num_retries: 4, since_last_attempt_s: 4, next_attempt_after_s: 55, message: HyperlaneMessage { id: 0x4b33454318f3e67fe733e7fe6c7a877eeca6a5e9fbb224c51eb183ad1dd61383, version: 3, nonce: 1, origin: test3, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x5d42ebdbba61412295d7b0302d6f50ac449ddb4f, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }, PendingMessage { num_retries: 4, since_last_attempt_s: 4, next_attempt_after_s: 55, message: HyperlaneMessage { id: 0xaf25f3da1da8917e47542818eb65fe6acefd3e2c8d74ecedcc4075bbe2669e7c, version: 3, nonce: 10, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0xd9fec8238711935d6c8d79bef2b9546ef23fc046, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }, PendingMessage { num_retries: 4, since_last_attempt_s: 4, next_attempt_after_s: 55, message: HyperlaneMessage { id: 0x300188dea2c304fa9b7d96e9f8983fd6cd1357b58853dd3124c5b8157299b345, version: 3, nonce: 9, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0xd9fec8238711935d6c8d79bef2b9546ef23fc046, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }, PendingMessage { num_retries: 4, since_last_attempt_s: 4, next_attempt_after_s: 55, message: HyperlaneMessage { id: 0x4bc6305e53882847f4a9993dbffc708cf20bb356973bda98ba4ddfbb1460733f, version: 3, nonce: 3, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x5d42ebdbba61412295d7b0302d6f50ac449ddb4f, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }] domain=test2 limit=5 queue_label=prepare_queue
2025-05-28T15:23:34.6197613Z <RLY> 2025-05-28T15:23:34.612219Z DEBUG prepare_classic_task:pop_many: relayer::msg::op_queue: Popped OpQueue operations queue_label=prepare_queue operations=[PendingMessage { num_retries: 1, since_last_attempt_s: 5, next_attempt_after_s: 0, message: HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }, PendingMessage { num_retries: 4, since_last_attempt_s: 5, next_attempt_after_s: 54, message: HyperlaneMessage { id: 0x4b33454318f3e67fe733e7fe6c7a877eeca6a5e9fbb224c51eb183ad1dd61383, version: 3, nonce: 1, origin: test3, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x5d42ebdbba61412295d7b0302d6f50ac449ddb4f, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }, PendingMessage { num_retries: 4, since_last_attempt_s: 5, next_attempt_after_s: 54, message: HyperlaneMessage { id: 0xaf25f3da1da8917e47542818eb65fe6acefd3e2c8d74ecedcc4075bbe2669e7c, version: 3, nonce: 10, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0xd9fec8238711935d6c8d79bef2b9546ef23fc046, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }, PendingMessage { num_retries: 4, since_last_attempt_s: 5, next_attempt_after_s: 54, message: HyperlaneMessage { id: 0x300188dea2c304fa9b7d96e9f8983fd6cd1357b58853dd3124c5b8157299b345, version: 3, nonce: 9, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0xd9fec8238711935d6c8d79bef2b9546ef23fc046, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }, PendingMessage { num_retries: 4, since_last_attempt_s: 5, next_attempt_after_s: 54, message: HyperlaneMessage { id: 0x4bc6305e53882847f4a9993dbffc708cf20bb356973bda98ba4ddfbb1460733f, version: 3, nonce: 3, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x5d42ebdbba61412295d7b0302d6f50ac449ddb4f, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None }] domain=test2 limit=5 queue_label=prepare_queue
2025-05-28T15:23:34.6222214Z <RLY> 2025-05-28T15:23:34.621567Z DEBUG prepare_classic_task:prepare: relayer::msg::gas_payment: Evaluating if message meets gas payment requirement hyp_message=HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 } policy=GasPaymentPolicyMinimum { minimum_payment: 1 } current_payment=InterchainGasPayment { message_id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, destination: 9913372, payment: 1050000000000000, gas_amount: 1050000 } current_expenditure=InterchainGasExpenditure { message_id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, tokens_used: 0, gas_used: 0 } tx_cost_estimate=TxCostEstimate { gas_limit: 0, gas_price: FixedPointNumber(BigDecimal(sign=NoSign, scale=0, digits=[])), l2_gas_limit: None } domain=test2 id=0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730
2025-05-28T15:23:34.6770388Z <RLY> 2025-05-28T15:23:34.676261Z  INFO prepare_classic_task:prepare:build:build:build:build: relayer::msg::metadata::multisig::base: List of validators and threshold for message hyp_message=HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 } validators=[0x00000000000000000000000015d34aaf54267db7d7c367839aaf71a00a2c6a65] threshold=1 domain=test2 id=0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730 ism_address=0x0000000000000000000000003ca8f9c04c7e3e1624ac2008f92f6f366a869444 destination_domain="test2" ism_address=0x0000000000000000000000003ca8f9c04c7e3e1624ac2008f92f6f366a869444 params=MessageMetadataBuildParams { ism_depth: 1, ism_count: Mutex { data: 1 } } ism_address=0x000000000000000000000000abaae0f6d24d9860b37b0aaf6c44d8b223b450cf destination_domain="test2" ism_address=0x000000000000000000000000abaae0f6d24d9860b37b0aaf6c44d8b223b450cf params=MessageMetadataBuildParams { ism_depth: 2, ism_count: Mutex { data: 2 } }
2025-05-28T15:23:34.6795248Z <RLY> 2025-05-28T15:23:34.678757Z DEBUG prepare_classic_task:prepare:build:build:build:build: relayer::msg::metadata::base_builder: List of validators and their storage locations for message hyp_message=HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 } validators=[0x00000000000000000000000015d34aaf54267db7d7c367839aaf71a00a2c6a65] validators_len=1 storage_locations=[["file:///tmp/.tmpJjM3l1"]] storage_locations_len=1 domain=test2 id=0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730 ism_address=0x0000000000000000000000003ca8f9c04c7e3e1624ac2008f92f6f366a869444 destination_domain="test2" ism_address=0x0000000000000000000000003ca8f9c04c7e3e1624ac2008f92f6f366a869444 params=MessageMetadataBuildParams { ism_depth: 1, ism_count: Mutex { data: 1 } } ism_address=0x000000000000000000000000abaae0f6d24d9860b37b0aaf6c44d8b223b450cf destination_domain="test2" ism_address=0x000000000000000000000000abaae0f6d24d9860b37b0aaf6c44d8b223b450cf params=MessageMetadataBuildParams { ism_depth: 2, ism_count: Mutex { data: 2 } }
2025-05-28T15:23:34.6804313Z <RLY> 2025-05-28T15:23:34.678942Z DEBUG prepare_classic_task:prepare:build:build:build:build: relayer::msg::metadata::base_builder: Validator and its storage locations for message hyp_message=HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 } validator=0x00000000000000000000000015d34aaf54267db7d7c367839aaf71a00a2c6a65 validator_storage_locations=["file:///tmp/.tmpJjM3l1"] domain=test2 id=0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730 ism_address=0x0000000000000000000000003ca8f9c04c7e3e1624ac2008f92f6f366a869444 destination_domain="test2" ism_address=0x0000000000000000000000003ca8f9c04c7e3e1624ac2008f92f6f366a869444 params=MessageMetadataBuildParams { ism_depth: 1, ism_count: Mutex { data: 1 } } ism_address=0x000000000000000000000000abaae0f6d24d9860b37b0aaf6c44d8b223b450cf destination_domain="test2" ism_address=0x000000000000000000000000abaae0f6d24d9860b37b0aaf6c44d8b223b450cf params=MessageMetadataBuildParams { ism_depth: 2, ism_count: Mutex { data: 2 } }
2025-05-28T15:23:34.6811740Z <RLY> 2025-05-28T15:23:34.679928Z DEBUG prepare_classic_task:prepare:build:build:build:build: hyperlane_base::types::multisig: Validator returned latest index validator=0x15d34aaf54267db7d7c367839aaf71a00a2c6a65 index=13 domain=test2 id=0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730 ism_address=0x0000000000000000000000003ca8f9c04c7e3e1624ac2008f92f6f366a869444 destination_domain="test2" ism_address=0x0000000000000000000000003ca8f9c04c7e3e1624ac2008f92f6f366a869444 params=MessageMetadataBuildParams { ism_depth: 1, ism_count: Mutex { data: 1 } } ism_address=0x000000000000000000000000abaae0f6d24d9860b37b0aaf6c44d8b223b450cf destination_domain="test2" ism_address=0x000000000000000000000000abaae0f6d24d9860b37b0aaf6c44d8b223b450cf params=MessageMetadataBuildParams { ism_depth: 2, ism_count: Mutex { data: 2 } }
2025-05-28T15:23:34.6829695Z <RLY> 2025-05-28T15:23:34.682405Z DEBUG prepare_classic_task:prepare:build:build:build:build:fetch_checkpoint: hyperlane_base::types::multisig: Found signed checkpoint validator="0x00000000000000000000000015d34aaf54267db7d7c367839aaf71a00a2c6a65" index=13 root="0x356862309e0c09e3b3058c9d2d0eeb84c552b417e4d4903616dc3ef395ff14d1" signature_count=1 domain=test2 id=0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730 ism_address=0x0000000000000000000000003ca8f9c04c7e3e1624ac2008f92f6f366a869444 destination_domain="test2" ism_address=0x0000000000000000000000003ca8f9c04c7e3e1624ac2008f92f6f366a869444 params=MessageMetadataBuildParams { ism_depth: 1, ism_count: Mutex { data: 1 } } ism_address=0x000000000000000000000000abaae0f6d24d9860b37b0aaf6c44d8b223b450cf destination_domain="test2" ism_address=0x000000000000000000000000abaae0f6d24d9860b37b0aaf6c44d8b223b450cf params=MessageMetadataBuildParams { ism_depth: 2, ism_count: Mutex { data: 2 } } validators=[0x00000000000000000000000015d34aaf54267db7d7c367839aaf71a00a2c6a65] threshold=1 index=13
2025-05-28T15:23:34.6839414Z <RLY> 2025-05-28T15:23:34.682473Z DEBUG prepare_classic_task:prepare:build:build:build:build:fetch_checkpoint: hyperlane_base::types::multisig: Fetched multisig checkpoint checkpoint=MultisigSignedCheckpoint { checkpoint: CheckpointWithMessageId { checkpoint: Checkpoint { merkle_tree_hook_address: 0x0000000000000000000000004826533b4897376654bb4d4ad88b7fafd0c98528, mailbox_domain: 9913371, root: 0x356862309e0c09e3b3058c9d2d0eeb84c552b417e4d4903616dc3ef395ff14d1, index: 13 }, message_id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730 }, signatures: [Signature { r: 17472094047143522713468587955132992787880489882299580370861649061322734836342, s: 26836275115876667154687350192202649268685518254175317394380184919756200859128, v: 27 }] } domain=test2 id=0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730 ism_address=0x0000000000000000000000003ca8f9c04c7e3e1624ac2008f92f6f366a869444 destination_domain="test2" ism_address=0x0000000000000000000000003ca8f9c04c7e3e1624ac2008f92f6f366a869444 params=MessageMetadataBuildParams { ism_depth: 1, ism_count: Mutex { data: 1 } } ism_address=0x000000000000000000000000abaae0f6d24d9860b37b0aaf6c44d8b223b450cf destination_domain="test2" ism_address=0x000000000000000000000000abaae0f6d24d9860b37b0aaf6c44d8b223b450cf params=MessageMetadataBuildParams { ism_depth: 2, ism_count: Mutex { data: 2 } } validators=[0x00000000000000000000000015d34aaf54267db7d7c367839aaf71a00a2c6a65] threshold=1 index=13
2025-05-28T15:23:34.6849712Z <RLY> 2025-05-28T15:23:34.683046Z DEBUG prepare_classic_task:prepare:build:build:build:build: relayer::msg::metadata::multisig::base: Found checkpoint with quorum hyp_message=HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 } metadata.checkpoint=CheckpointWithMessageId { checkpoint: Checkpoint { merkle_tree_hook_address: 0x0000000000000000000000004826533b4897376654bb4d4ad88b7fafd0c98528, mailbox_domain: 9913371, root: 0x356862309e0c09e3b3058c9d2d0eeb84c552b417e4d4903616dc3ef395ff14d1, index: 13 }, message_id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730 } domain=test2 id=0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730 ism_address=0x0000000000000000000000003ca8f9c04c7e3e1624ac2008f92f6f366a869444 destination_domain="test2" ism_address=0x0000000000000000000000003ca8f9c04c7e3e1624ac2008f92f6f366a869444 params=MessageMetadataBuildParams { ism_depth: 1, ism_count: Mutex { data: 1 } } ism_address=0x000000000000000000000000abaae0f6d24d9860b37b0aaf6c44d8b223b450cf destination_domain="test2" ism_address=0x000000000000000000000000abaae0f6d24d9860b37b0aaf6c44d8b223b450cf params=MessageMetadataBuildParams { ism_depth: 2, ism_count: Mutex { data: 2 } }
2025-05-28T15:23:34.6907524Z <RLY> 2025-05-28T15:23:34.690317Z  INFO prepare_classic_task:prepare:build:build:build:build: relayer::msg::metadata::aggregation: Built metadata using fast path domain=test2 id=0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730 ism_address=0x0000000000000000000000003ca8f9c04c7e3e1624ac2008f92f6f366a869444 destination_domain="test2" ism_address=0x0000000000000000000000003ca8f9c04c7e3e1624ac2008f92f6f366a869444 params=MessageMetadataBuildParams { ism_depth: 1, ism_count: Mutex { data: 1 } } ism_address=0x000000000000000000000000abaae0f6d24d9860b37b0aaf6c44d8b223b450cf destination_domain="test2" ism_address=0x000000000000000000000000abaae0f6d24d9860b37b0aaf6c44d8b223b450cf params=MessageMetadataBuildParams { ism_depth: 2, ism_count: Mutex { data: 2 } }
2025-05-28T15:23:34.6914888Z <RLY> 2025-05-28T15:23:34.690359Z  INFO prepare_classic_task:prepare:build:build:build:build: relayer::msg::metadata::aggregation: return=Metadata([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 38, 83, 59, 72, 151, 55, 102, 84, 187, 77, 74, 216, 139, 127, 175, 208, 201, 133, 40, 53, 104, 98, 48, 158, 12, 9, 227, 179, 5, 140, 157, 45, 14, 235, 132, 197, 82, 180, 23, 228, 212, 144, 54, 22, 220, 62, 243, 149, 255, 20, 209, 0, 0, 0, 13, 38, 160, 218, 211, 12, 240, 20, 229, 48, 22, 213, 123, 127, 198, 24, 94, 144, 178, 53, 129, 249, 254, 121, 28, 16, 162, 235, 57, 133, 162, 26, 118, 59, 84, 203, 31, 54, 167, 160, 18, 98, 98, 118, 40, 119, 152, 69, 250, 91, 200, 89, 87, 57, 44, 13, 116, 176, 55, 189, 25, 107, 253, 213, 248, 27]) domain=test2 id=0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730 ism_address=0x0000000000000000000000003ca8f9c04c7e3e1624ac2008f92f6f366a869444 destination_domain="test2" ism_address=0x0000000000000000000000003ca8f9c04c7e3e1624ac2008f92f6f366a869444 params=MessageMetadataBuildParams { ism_depth: 1, ism_count: Mutex { data: 1 } } ism_address=0x000000000000000000000000abaae0f6d24d9860b37b0aaf6c44d8b223b450cf destination_domain="test2" ism_address=0x000000000000000000000000abaae0f6d24d9860b37b0aaf6c44d8b223b450cf params=MessageMetadataBuildParams { ism_depth: 2, ism_count: Mutex { data: 2 } }
2025-05-28T15:23:34.6921474Z <RLY> 2025-05-28T15:23:34.691353Z  INFO prepare_classic_task:prepare:build:build: relayer::msg::metadata::routing: return=Metadata([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 38, 83, 59, 72, 151, 55, 102, 84, 187, 77, 74, 216, 139, 127, 175, 208, 201, 133, 40, 53, 104, 98, 48, 158, 12, 9, 227, 179, 5, 140, 157, 45, 14, 235, 132, 197, 82, 180, 23, 228, 212, 144, 54, 22, 220, 62, 243, 149, 255, 20, 209, 0, 0, 0, 13, 38, 160, 218, 211, 12, 240, 20, 229, 48, 22, 213, 123, 127, 198, 24, 94, 144, 178, 53, 129, 249, 254, 121, 28, 16, 162, 235, 57, 133, 162, 26, 118, 59, 84, 203, 31, 54, 167, 160, 18, 98, 98, 118, 40, 119, 152, 69, 250, 91, 200, 89, 87, 57, 44, 13, 116, 176, 55, 189, 25, 107, 253, 213, 248, 27]) domain=test2 id=0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730 ism_address=0x0000000000000000000000003ca8f9c04c7e3e1624ac2008f92f6f366a869444 destination_domain="test2" ism_address=0x0000000000000000000000003ca8f9c04c7e3e1624ac2008f92f6f366a869444 params=MessageMetadataBuildParams { ism_depth: 1, ism_count: Mutex { data: 1 } }
2025-05-28T15:23:34.7028793Z <RLY> 2025-05-28T15:23:34.702290Z DEBUG prepare_classic_task:prepare:process_estimate_costs: hyperlane_ethereum::tx: Gas limit set for transaction estimated_gas_limit=105400 gas_override=None used_gas_limit=105400 domain=test2 id=0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730 HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 } msg=HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 } metadata=0x000000000000000000000010000000950000000000000000000000004826533b4897376654bb4d4ad88b7fafd0c98528356862309e0c09e3b3058c9d2d0eeb84c552b417e4d4903616dc3ef395ff14d10000000d26a0dad30cf014e53016d57b7fc6185e90b23581f9fe791c10a2eb3985a21a763b54cb1f36a7a01262627628779845fa5bc85957392c0d74b037bd196bfdd5f81b
2025-05-28T15:23:34.7102817Z <RLY> 2025-05-28T15:23:34.709624Z DEBUG prepare_classic_task:prepare: relayer::msg::gas_payment: Evaluating if message meets gas payment requirement hyp_message=HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 } policy=GasPaymentPolicyMinimum { minimum_payment: 1 } current_payment=InterchainGasPayment { message_id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, destination: 9913372, payment: 1050000000000000, gas_amount: 1050000 } current_expenditure=InterchainGasExpenditure { message_id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, tokens_used: 0, gas_used: 0 } tx_cost_estimate=TxCostEstimate { gas_limit: 105400, gas_price: FixedPointNumber(BigDecimal(sign=Plus, scale=0, digits=[1000000007])), l2_gas_limit: None } domain=test2 id=0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730
2025-05-28T15:23:34.7107528Z <RLY> 2025-05-28T15:23:34.709826Z DEBUG prepare_classic_task:prepare: relayer::msg::pending_message: Gas payment requirement met, ready to process message gas_limit=105400 tx_cost_estimate=TxCostEstimate { gas_limit: 105400, gas_price: FixedPointNumber(BigDecimal(sign=Plus, scale=0, digits=[1000000007])), l2_gas_limit: None } domain=test2 id=0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730
2025-05-28T15:23:34.7111012Z <RLY> 2025-05-28T15:23:34.709926Z DEBUG prepare_classic_task: relayer::msg::op_submitter: Operation prepared op=PendingMessage { num_retries: 1, since_last_attempt_s: 5, next_attempt_after_s: 0, message: HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 }, status: Retry(CouldNotFetchMetadata), app_context: None } domain=test2
2025-05-28T15:23:34.7505206Z <RLY> 2025-05-28T15:23:34.750003Z DEBUG submit_classic_task:pop_many: relayer::msg::op_queue: Popped OpQueue operations queue_label=submit_queue operations=[PendingMessage { num_retries: 1, since_last_attempt_s: 5, next_attempt_after_s: 0, message: HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 }, status: ReadyToSubmit, app_context: None }] domain=test2 limit=5 queue_label=submit_queue
2025-05-28T15:23:34.7615146Z <RLY> 2025-05-28T15:23:34.760721Z DEBUG submit_classic_task:submit_single_operation:submit:process_estimate_costs: hyperlane_ethereum::tx: Gas limit set for transaction estimated_gas_limit=105400 gas_override=None used_gas_limit=105400 domain=test2 op=PendingMessage { num_retries: 1, since_last_attempt_s: 5, next_attempt_after_s: 0, message: HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 }, status: ReadyToSubmit, app_context: None } id=0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730 domain=test2 HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 } msg=HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 } metadata=0x000000000000000000000010000000950000000000000000000000004826533b4897376654bb4d4ad88b7fafd0c98528356862309e0c09e3b3058c9d2d0eeb84c552b417e4d4903616dc3ef395ff14d10000000d26a0dad30cf014e53016d57b7fc6185e90b23581f9fe791c10a2eb3985a21a763b54cb1f36a7a01262627628779845fa5bc85957392c0d74b037bd196bfdd5f81b
2025-05-28T15:23:34.7695188Z <RLY> 2025-05-28T15:23:34.768965Z DEBUG submit_classic_task:submit_single_operation:submit:process: hyperlane_ethereum::tx: Gas limit set for transaction estimated_gas_limit=180400 gas_override=None used_gas_limit=180400 domain=test2 op=PendingMessage { num_retries: 1, since_last_attempt_s: 5, next_attempt_after_s: 0, message: HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 }, status: ReadyToSubmit, app_context: None } id=0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730 domain=test2 tx_gas_limit=Some(105400) metadata=0x000000000000000000000010000000950000000000000000000000004826533b4897376654bb4d4ad88b7fafd0c98528356862309e0c09e3b3058c9d2d0eeb84c552b417e4d4903616dc3ef395ff14d10000000d26a0dad30cf014e53016d57b7fc6185e90b23581f9fe791c10a2eb3985a21a763b54cb1f36a7a01262627628779845fa5bc85957392c0d74b037bd196bfdd5f81b
2025-05-28T15:23:34.7736715Z <RLY> 2025-05-28T15:23:34.773103Z  INFO submit_classic_task:submit_single_operation:submit:process: hyperlane_ethereum::tx: Dispatching transaction to=0x7bc06c482dead17c0e297afbc32f6e63d3846650 from=None gas_limit=Some(180400) gas_price=Some(200000) nonce=None domain=test2 op=PendingMessage { num_retries: 1, since_last_attempt_s: 5, next_attempt_after_s: 0, message: HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 }, status: ReadyToSubmit, app_context: None } id=0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730 domain=test2 tx_gas_limit=Some(105400) metadata=0x000000000000000000000010000000950000000000000000000000004826533b4897376654bb4d4ad88b7fafd0c98528356862309e0c09e3b3058c9d2d0eeb84c552b417e4d4903616dc3ef395ff14d10000000d26a0dad30cf014e53016d57b7fc6185e90b23581f9fe791c10a2eb3985a21a763b54cb1f36a7a01262627628779845fa5bc85957392c0d74b037bd196bfdd5f81b
2025-05-28T15:23:34.7754463Z <RLY> 2025-05-28T15:23:34.774937Z DEBUG submit_classic_task:submit_single_operation:submit:process:NonceManager::send_transaction: ethers_middleware::nonce_manager: Sending transaction nonce=Some(4) domain=test2 op=PendingMessage { num_retries: 1, since_last_attempt_s: 5, next_attempt_after_s: 0, message: HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 }, status: ReadyToSubmit, app_context: None } id=0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730 domain=test2 tx_gas_limit=Some(105400) metadata=0x000000000000000000000010000000950000000000000000000000004826533b4897376654bb4d4ad88b7fafd0c98528356862309e0c09e3b3058c9d2d0eeb84c552b417e4d4903616dc3ef395ff14d10000000d26a0dad30cf014e53016d57b7fc6185e90b23581f9fe791c10a2eb3985a21a763b54cb1f36a7a01262627628779845fa5bc85957392c0d74b037bd196bfdd5f81b
2025-05-28T15:23:34.7766644Z <RLY> 2025-05-28T15:23:34.775017Z DEBUG submit_classic_task:submit_single_operation:submit:process:NonceManager::send_transaction:GasOracle::send_transaction: ethers_middleware::gas_oracle::middleware: Sending transaction tx=Eip1559(Eip1559TransactionRequest { from: None, to: Some(0x7bc06c482dead17c0e297afbc32f6e63d3846650), gas: Some(180400), value: None, data: Some(Bytes(0x7c39d130000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000095000000000000000000000010000000950000000000000000000000004826533b4897376654bb4d4ad88b7fafd0c98528356862309e0c09e3b3058c9d2d0eeb84c552b417e4d4903616dc3ef395ff14d10000000d26a0dad30cf014e53016d57b7fc6185e90b23581f9fe791c10a2eb3985a21a763b54cb1f36a7a01262627628779845fa5bc85957392c0d74b037bd196bfdd5f81b0000000000000000000000000000000000000000000000000000000000000000000000000000000000004f030000000d0097441b000000000000000000000000f39fd6e51aad88f6f4ce6ab8827279cfffb922660097441c0000000000000000000000004951a1c579039ebfcba0be33d2cd3a6d30b0f80212340000000000000000000000000000000000)), nonce: Some(4), access_list: AccessList([]), max_priority_fee_per_gas: Some(100000), max_fee_per_gas: Some(200000), chain_id: None }) domain=test2 op=PendingMessage { num_retries: 1, since_last_attempt_s: 5, next_attempt_after_s: 0, message: HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 }, status: ReadyToSubmit, app_context: None } id=0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730 domain=test2 tx_gas_limit=Some(105400) metadata=0x000000000000000000000010000000950000000000000000000000004826533b4897376654bb4d4ad88b7fafd0c98528356862309e0c09e3b3058c9d2d0eeb84c552b417e4d4903616dc3ef395ff14d10000000d26a0dad30cf014e53016d57b7fc6185e90b23581f9fe791c10a2eb3985a21a763b54cb1f36a7a01262627628779845fa5bc85957392c0d74b037bd196bfdd5f81b
2025-05-28T15:23:34.7786774Z <RLY> 2025-05-28T15:23:34.775208Z DEBUG submit_classic_task:submit_single_operation:submit:process:NonceManager::send_transaction:GasOracle::send_transaction:GasOracle::fill_transaction: ethers_middleware::gas_oracle::middleware: Filled transaction tx=Eip1559(Eip1559TransactionRequest { from: None, to: Some(0x7bc06c482dead17c0e297afbc32f6e63d3846650), gas: Some(180400), value: None, data: Some(Bytes(0x7c39d130000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000095000000000000000000000010000000950000000000000000000000004826533b4897376654bb4d4ad88b7fafd0c98528356862309e0c09e3b3058c9d2d0eeb84c552b417e4d4903616dc3ef395ff14d10000000d26a0dad30cf014e53016d57b7fc6185e90b23581f9fe791c10a2eb3985a21a763b54cb1f36a7a01262627628779845fa5bc85957392c0d74b037bd196bfdd5f81b0000000000000000000000000000000000000000000000000000000000000000000000000000000000004f030000000d0097441b000000000000000000000000f39fd6e51aad88f6f4ce6ab8827279cfffb922660097441c0000000000000000000000004951a1c579039ebfcba0be33d2cd3a6d30b0f80212340000000000000000000000000000000000)), nonce: Some(4), access_list: AccessList([]), max_priority_fee_per_gas: Some(100000), max_fee_per_gas: Some(200000), chain_id: None }) domain=test2 op=PendingMessage { num_retries: 1, since_last_attempt_s: 5, next_attempt_after_s: 0, message: HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 }, status: ReadyToSubmit, app_context: None } id=0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730 domain=test2 tx_gas_limit=Some(105400) metadata=0x000000000000000000000010000000950000000000000000000000004826533b4897376654bb4d4ad88b7fafd0c98528356862309e0c09e3b3058c9d2d0eeb84c552b417e4d4903616dc3ef395ff14d10000000d26a0dad30cf014e53016d57b7fc6185e90b23581f9fe791c10a2eb3985a21a763b54cb1f36a7a01262627628779845fa5bc85957392c0d74b037bd196bfdd5f81b tx=Eip1559(Eip1559TransactionRequest { from: None, to: Some(0x7bc06c482dead17c0e297afbc32f6e63d3846650), gas: Some(180400), value: None, data: Some(Bytes(0x7c39d130000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000095000000000000000000000010000000950000000000000000000000004826533b4897376654bb4d4ad88b7fafd0c98528356862309e0c09e3b3058c9d2d0eeb84c552b417e4d4903616dc3ef395ff14d10000000d26a0dad30cf014e53016d57b7fc6185e90b23581f9fe791c10a2eb3985a21a763b54cb1f36a7a01262627628779845fa5bc85957392c0d74b037bd196bfdd5f81b0000000000000000000000000000000000000000000000000000000000000000000000000000000000004f030000000d0097441b000000000000000000000000f39fd6e51aad88f6f4ce6ab8827279cfffb922660097441c0000000000000000000000004951a1c579039ebfcba0be33d2cd3a6d30b0f80212340000000000000000000000000000000000)), nonce: Some(4), access_list: AccessList([]), max_priority_fee_per_gas: Some(100000), max_fee_per_gas: Some(200000), chain_id: None }) block=None
2025-05-28T15:23:34.7801253Z <RLY> 2025-05-28T15:23:34.776358Z DEBUG submit_classic_task:submit_single_operation:submit:process:NonceManager::send_transaction:GasOracle::send_transaction:GasOracle::send_transaction:SignerMiddleware::send_transaction: ethers_middleware::signer: Sending transaction nonce=Some(4) domain=test2 op=PendingMessage { num_retries: 1, since_last_attempt_s: 5, next_attempt_after_s: 0, message: HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 }, status: ReadyToSubmit, app_context: None } id=0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730 domain=test2 tx_gas_limit=Some(105400) metadata=0x000000000000000000000010000000950000000000000000000000004826533b4897376654bb4d4ad88b7fafd0c98528356862309e0c09e3b3058c9d2d0eeb84c552b417e4d4903616dc3ef395ff14d10000000d26a0dad30cf014e53016d57b7fc6185e90b23581f9fe791c10a2eb3985a21a763b54cb1f36a7a01262627628779845fa5bc85957392c0d74b037bd196bfdd5f81b
2025-05-28T15:23:34.7815042Z <RLY> 2025-05-28T15:23:34.780409Z DEBUG submit_classic_task:submit_single_operation:submit:process:NonceManager::send_transaction:GasOracle::send_transaction:GasOracle::send_transaction: ethers_middleware::gas_escalator: Sent tx, adding to gas escalator watcher tx=Eip1559(Eip1559TransactionRequest { from: Some(0x23618e81e3f5cdf7f54c3d65f7fbc0abf5b21e8f), to: Some(0x7bc06c482dead17c0e297afbc32f6e63d3846650), gas: Some(180400), value: None, data: Some(Bytes(0x7c39d130000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000095000000000000000000000010000000950000000000000000000000004826533b4897376654bb4d4ad88b7fafd0c98528356862309e0c09e3b3058c9d2d0eeb84c552b417e4d4903616dc3ef395ff14d10000000d26a0dad30cf014e53016d57b7fc6185e90b23581f9fe791c10a2eb3985a21a763b54cb1f36a7a01262627628779845fa5bc85957392c0d74b037bd196bfdd5f81b0000000000000000000000000000000000000000000000000000000000000000000000000000000000004f030000000d0097441b000000000000000000000000f39fd6e51aad88f6f4ce6ab8827279cfffb922660097441c0000000000000000000000004951a1c579039ebfcba0be33d2cd3a6d30b0f80212340000000000000000000000000000000000)), nonce: Some(4), access_list: AccessList([]), max_priority_fee_per_gas: Some(100000), max_fee_per_gas: Some(200000), chain_id: Some(31337) }) tx_hash=0x2be1476cdc070661d058712cea6b497fc6363372c8d614e981ad8f2541c43d98 domain=test2 op=PendingMessage { num_retries: 1, since_last_attempt_s: 5, next_attempt_after_s: 0, message: HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 }, status: ReadyToSubmit, app_context: None } id=0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730 domain=test2 tx_gas_limit=Some(105400) metadata=0x000000000000000000000010000000950000000000000000000000004826533b4897376654bb4d4ad88b7fafd0c98528356862309e0c09e3b3058c9d2d0eeb84c552b417e4d4903616dc3ef395ff14d10000000d26a0dad30cf014e53016d57b7fc6185e90b23581f9fe791c10a2eb3985a21a763b54cb1f36a7a01262627628779845fa5bc85957392c0d74b037bd196bfdd5f81b
2025-05-28T15:23:34.7826272Z <RLY> 2025-05-28T15:23:34.781376Z DEBUG submit_classic_task:submit_single_operation:submit:process:NonceManager::send_transaction: ethers_middleware::nonce_manager: Sent transaction nonce=Some(4) domain=test2 op=PendingMessage { num_retries: 1, since_last_attempt_s: 5, next_attempt_after_s: 0, message: HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 }, status: ReadyToSubmit, app_context: None } id=0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730 domain=test2 tx_gas_limit=Some(105400) metadata=0x000000000000000000000010000000950000000000000000000000004826533b4897376654bb4d4ad88b7fafd0c98528356862309e0c09e3b3058c9d2d0eeb84c552b417e4d4903616dc3ef395ff14d10000000d26a0dad30cf014e53016d57b7fc6185e90b23581f9fe791c10a2eb3985a21a763b54cb1f36a7a01262627628779845fa5bc85957392c0d74b037bd196bfdd5f81b
2025-05-28T15:23:34.7833954Z <RLY> 2025-05-28T15:23:34.781408Z DEBUG submit_classic_task:submit_single_operation:submit:process:NonceManager::send_transaction: ethers_middleware::nonce_manager: Transactions until nonce resync txs_until_resync=9 domain=test2 op=PendingMessage { num_retries: 1, since_last_attempt_s: 5, next_attempt_after_s: 0, message: HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 }, status: ReadyToSubmit, app_context: None } id=0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730 domain=test2 tx_gas_limit=Some(105400) metadata=0x000000000000000000000010000000950000000000000000000000004826533b4897376654bb4d4ad88b7fafd0c98528356862309e0c09e3b3058c9d2d0eeb84c552b417e4d4903616dc3ef395ff14d10000000d26a0dad30cf014e53016d57b7fc6185e90b23581f9fe791c10a2eb3985a21a763b54cb1f36a7a01262627628779845fa5bc85957392c0d74b037bd196bfdd5f81b
2025-05-28T15:23:34.7841757Z <RLY> 2025-05-28T15:23:34.781948Z  INFO submit_classic_task:submit_single_operation:submit:process:track_pending_tx: hyperlane_ethereum::tx: Dispatched tx tx_hash=0x2be1476cdc070661d058712cea6b497fc6363372c8d614e981ad8f2541c43d98 domain=test2 op=PendingMessage { num_retries: 1, since_last_attempt_s: 5, next_attempt_after_s: 0, message: HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 }, status: ReadyToSubmit, app_context: None } id=0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730 domain=test2 tx_gas_limit=Some(105400) metadata=0x000000000000000000000010000000950000000000000000000000004826533b4897376654bb4d4ad88b7fafd0c98528356862309e0c09e3b3058c9d2d0eeb84c552b417e4d4903616dc3ef395ff14d10000000d26a0dad30cf014e53016d57b7fc6185e90b23581f9fe791c10a2eb3985a21a763b54cb1f36a7a01262627628779845fa5bc85957392c0d74b037bd196bfdd5f81b
2025-05-28T15:23:36.7827242Z <RLY> 2025-05-28T15:23:36.782099Z DEBUG submit_classic_task:submit_single_operation:submit:process:track_pending_tx: ethers_providers::pending_transaction: Starting to poll pending tx 0x2be1476cdc070661d058712cea6b497fc6363372c8d614e981ad8f2541c43d98 domain=test2 op=PendingMessage { num_retries: 1, since_last_attempt_s: 5, next_attempt_after_s: 0, message: HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 }, status: ReadyToSubmit, app_context: None } id=0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730 domain=test2 tx_gas_limit=Some(105400) metadata=0x000000000000000000000010000000950000000000000000000000004826533b4897376654bb4d4ad88b7fafd0c98528356862309e0c09e3b3058c9d2d0eeb84c552b417e4d4903616dc3ef395ff14d10000000d26a0dad30cf014e53016d57b7fc6185e90b23581f9fe791c10a2eb3985a21a763b54cb1f36a7a01262627628779845fa5bc85957392c0d74b037bd196bfdd5f81b
2025-05-28T15:23:36.7863268Z <RLY> 2025-05-28T15:23:36.785766Z DEBUG submit_classic_task:submit_single_operation:submit:process:track_pending_tx: ethers_providers::pending_transaction: Getting receipt for pending tx 0x2be1476cdc070661d058712cea6b497fc6363372c8d614e981ad8f2541c43d98 domain=test2 op=PendingMessage { num_retries: 1, since_last_attempt_s: 5, next_attempt_after_s: 0, message: HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 }, status: ReadyToSubmit, app_context: None } id=0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730 domain=test2 tx_gas_limit=Some(105400) metadata=0x000000000000000000000010000000950000000000000000000000004826533b4897376654bb4d4ad88b7fafd0c98528356862309e0c09e3b3058c9d2d0eeb84c552b417e4d4903616dc3ef395ff14d10000000d26a0dad30cf014e53016d57b7fc6185e90b23581f9fe791c10a2eb3985a21a763b54cb1f36a7a01262627628779845fa5bc85957392c0d74b037bd196bfdd5f81b
2025-05-28T15:23:36.7890942Z <RLY> 2025-05-28T15:23:36.788536Z DEBUG submit_classic_task:submit_single_operation:submit:process:track_pending_tx: ethers_providers::pending_transaction: Checking receipt for pending tx 0x2be1476cdc070661d058712cea6b497fc6363372c8d614e981ad8f2541c43d98 domain=test2 op=PendingMessage { num_retries: 1, since_last_attempt_s: 5, next_attempt_after_s: 0, message: HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 }, status: ReadyToSubmit, app_context: None } id=0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730 domain=test2 tx_gas_limit=Some(105400) metadata=0x000000000000000000000010000000950000000000000000000000004826533b4897376654bb4d4ad88b7fafd0c98528356862309e0c09e3b3058c9d2d0eeb84c552b417e4d4903616dc3ef395ff14d10000000d26a0dad30cf014e53016d57b7fc6185e90b23581f9fe791c10a2eb3985a21a763b54cb1f36a7a01262627628779845fa5bc85957392c0d74b037bd196bfdd5f81b
2025-05-28T15:23:36.7898904Z <RLY> 2025-05-28T15:23:36.788681Z  INFO submit_classic_task:submit_single_operation:submit:process:track_pending_tx: hyperlane_ethereum::tx: confirmed transaction tx_hash=0x2be1476cdc070661d058712cea6b497fc6363372c8d614e981ad8f2541c43d98 domain=test2 op=PendingMessage { num_retries: 1, since_last_attempt_s: 5, next_attempt_after_s: 0, message: HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 }, status: ReadyToSubmit, app_context: None } id=0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730 domain=test2 tx_gas_limit=Some(105400) metadata=0x000000000000000000000010000000950000000000000000000000004826533b4897376654bb4d4ad88b7fafd0c98528356862309e0c09e3b3058c9d2d0eeb84c552b417e4d4903616dc3ef395ff14d10000000d26a0dad30cf014e53016d57b7fc6185e90b23581f9fe791c10a2eb3985a21a763b54cb1f36a7a01262627628779845fa5bc85957392c0d74b037bd196bfdd5f81b
2025-05-28T15:23:36.7907165Z <RLY> 2025-05-28T15:23:36.789999Z DEBUG submit_classic_task:submit_single_operation:submit: relayer::msg::gas_payment: Recording gas expenditure for message hyp_message=HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 } outcome=TxOutcome { transaction_id: 0x00000000000000000000000000000000000000000000000000000000000000002be1476cdc070661d058712cea6b497fc6363372c8d614e981ad8f2541c43d98, executed: true, gas_used: 104289, gas_price: FixedPointNumber(BigDecimal(sign=Plus, scale=0, digits=[100007])) } domain=test2 op=PendingMessage { num_retries: 1, since_last_attempt_s: 5, next_attempt_after_s: 0, message: HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 }, status: ReadyToSubmit, app_context: None } id=0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730 domain=test2
2025-05-28T15:23:36.7914737Z <RLY> 2025-05-28T15:23:36.790452Z DEBUG submit_classic_task:submit_single_operation:submit: hyperlane_base::db::rocks::hyperlane_db: Storing gas expenditure event=InterchainGasExpenditure { message_id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, tokens_used: 10429630023, gas_used: 104289 } new_total_gas_expenditure=InterchainGasExpenditure { message_id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, tokens_used: 10429630023, gas_used: 104289 } domain=test2 op=PendingMessage { num_retries: 1, since_last_attempt_s: 5, next_attempt_after_s: 0, message: HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 }, status: ReadyToSubmit, app_context: None } id=0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730 domain=test2
2025-05-28T15:23:36.7921909Z <RLY> 2025-05-28T15:23:36.790636Z DEBUG submit_classic_task:submit_single_operation:submit: relayer::msg::pending_message: Gas used by message submission actual_gas_for_message=104289 message_gas_estimate=105400 submission_gas_estimate=105400 hyp_message=HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 } domain=test2 op=PendingMessage { num_retries: 1, since_last_attempt_s: 5, next_attempt_after_s: 0, message: HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 }, status: ReadyToSubmit, app_context: None } id=0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730 domain=test2
2025-05-28T15:23:36.7928535Z <RLY> 2025-05-28T15:23:36.791264Z DEBUG submit_classic_task:submit_single_operation: relayer::msg::op_submitter: Operation submitted op=PendingMessage { num_retries: 1, since_last_attempt_s: 7, next_attempt_after_s: 0, message: HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 }, status: ReadyToSubmit, app_context: None } domain=test2 op=PendingMessage { num_retries: 1, since_last_attempt_s: 5, next_attempt_after_s: 0, message: HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 }, status: ReadyToSubmit, app_context: None }
2025-05-28T15:23:36.7933395Z <RLY> 2025-05-28T15:23:36.791923Z DEBUG submit_classic_task:submit_single_operation: relayer::msg::op_submitter: return=() domain=test2 op=PendingMessage { num_retries: 1, since_last_attempt_s: 5, next_attempt_after_s: 0, message: HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 }, status: ReadyToSubmit, app_context: None }
2025-05-28T15:23:36.9297387Z <RLY> 2025-05-28T15:23:36.929145Z DEBUG confirm_classic_task:pop_many: relayer::msg::op_queue: Popped OpQueue operations queue_label=confirm_queue operations=[PendingMessage { num_retries: 1, since_last_attempt_s: 7, next_attempt_after_s: 4, message: HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 }, status: Confirm(SubmittedBySelf), app_context: None }] domain=test2 limit=5 queue_label=confirm_queue
2025-05-28T15:23:37.4310476Z <RLY> 2025-05-28T15:23:37.430449Z DEBUG confirm_classic_task:pop_many: relayer::msg::op_queue: Popped OpQueue operations queue_label=confirm_queue operations=[PendingMessage { num_retries: 1, since_last_attempt_s: 7, next_attempt_after_s: 4, message: HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 }, status: Confirm(SubmittedBySelf), app_context: None }] domain=test2 limit=5 queue_label=confirm_queue
2025-05-28T15:23:37.9319938Z <RLY> 2025-05-28T15:23:37.931429Z DEBUG confirm_classic_task:pop_many: relayer::msg::op_queue: Popped OpQueue operations queue_label=confirm_queue operations=[PendingMessage { num_retries: 1, since_last_attempt_s: 8, next_attempt_after_s: 3, message: HyperlaneMessage { id: 0x4a75f9fd941717c7e0ef105803f3c1fdb1f2f4a526dab30815db46b6f7124730, version: 3, nonce: 13, origin: test1, sender: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266, destination: test2, recipient: 0x4951a1c579039ebfcba0be33d2cd3a6d30b0f802, body: 0x1234 }, status: Confirm(SubmittedBySelf), app_context: None }] domain=test2 limit=5 queue_label=confirm_queue
```

author_association: CONTRIBUTOR