Repository: FuelLabs/fuel-core (Rust)

Issue:
Ensure that L1 Events received from the relayer are in a deterministic order
We are now writing the merkle root of all the events received via the relayer to the header of each block. It is important that the order of those events is the same for all nodes verifying blocks so that the merkle root is the same.

Related to https://github.com/FuelLabs/fuel-core/pull/1769



Fix (patch):
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7fa057f60db..bec0733ed1a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -23,6 +23,7 @@ Description of the upcoming release here.
 ### Changed
 
 #### Breaking
+- [1779](https://github.com/FuelLabs/fuel-core/pull/1779): Modify Relayer service to order Events from L1 by block index
 - [#1783](https://github.com/FuelLabs/fuel-core/pull/1783): The PR upgrade `fuel-vm` to `0.48.0` release. Because of some breaking changes, we also adapted our codebase to follow them: 
   - Implementation of `Default` for configs was moved under the `test-helpers` feature. The `fuel-core` binary uses testnet configuration instead of `Default::default`(for cases when `ChainConfig` was not provided by the user).
   - All parameter types are enums now and require corresponding modifications across the codebase(we need to use getters and setters). The GraphQL API remains the same for simplicity, but each parameter now has one more field - `version`, that can be used to decide how to deserialize. 
diff --git a/crates/services/relayer/src/mock_db.rs b/crates/services/relayer/src/mock_db.rs
index 7e307a6a0c5..1c8c512c4d2 100644
--- a/crates/services/relayer/src/mock_db.rs
+++ b/crates/services/relayer/src/mock_db.rs
@@ -16,10 +16,7 @@ use fuel_core_types::{
     services::relayer::Event,
 };
 use std::{
-    collections::{
-        BTreeMap,
-        HashMap,
-    },
+    collections::BTreeMap,
     sync::{
         Arc,
         Mutex,
@@ -28,9 +25,9 @@ use std::{
 
 #[derive(Default)]
 pub struct Data {
-    pub messages: BTreeMap<DaBlockHeight, HashMap<Nonce, Message>>,
+    pub messages: BTreeMap<DaBlockHeight, Vec<(Nonce, Message)>>,
     pub transactions:
-        BTreeMap<DaBlockHeight, HashMap<RelayedTransactionId, RelayedTransaction>>,
+        BTreeMap<DaBlockHeight, Vec<(RelayedTransactionId, RelayedTransaction)>>,
     pub finalized_da_height: Option<DaBlockHeight>,
 }
 
@@ -44,13 +41,27 @@ pub struct MockDb {
 }
 
 impl MockDb {
-    pub fn get_message(&self, id: &Nonce) -> Option<Message> {
+    pub fn get_message(&self, nonce: &Nonce) -> Option<Message> {
         self.data
             .lock()
             .unwrap()
             .messages
             .iter()
-            .find_map(|(_, map)| map.get(id).cloned())
+            .find_map(|(_, map)| {
+                map.iter()
+                    .find(|(inner_nonce, _msg)| nonce == inner_nonce)
+                    .map(|(_, msg)| msg.clone())
+            })
+    }
+
+    pub fn get_messages_for_block(&self, da_block_height: DaBlockHeight) -> Vec<Message> {
+        self.data
+            .lock()
+            .unwrap()
+            .messages
+            .get(&da_block_height)
+            .map(|map| map.iter().map(|(_, msg)| msg).cloned().collect())
+            .unwrap_or_default()
     }
 
     pub fn get_transaction(
@@ -62,7 +73,24 @@ impl MockDb {
             .unwrap()
             .transactions
             .iter()
-            .find_map(|(_, map)| map.get(id).cloned())
+            .find_map(|(_, txs)| {
+                txs.iter()
+                    .find(|(inner_id, _tx)| id == inner_id)
+                    .map(|(_, tx)| tx.clone())
+            })
+    }
+
+    pub fn get_transactions_for_block(
+        &self,
+        da_block_height: DaBlockHeight,
+    ) -> Vec<RelayedTransaction> {
+        self.data
+            .lock()
+            .unwrap()
+            .transactions
+            .get(&da_block_height)
+            .map(|map| map.iter().map(|(_, tx)| tx).cloned().collect())
+            .unwrap_or_default()
     }
 }
 
@@ -79,13 +107,13 @@ impl RelayerDb for MockDb {
                     m.messages
                         .entry(message.da_height())
                         .or_default()
-                        .insert(*message.id(), message.clone());
+                        .push((*message.id(), message.clone()));
                 }
                 Event::Transaction(transaction) => {
                     m.transactions
                         .entry(transaction.da_height())
                         .or_default()
-                        .insert(transaction.id(), transaction.clone());
+                        .push((transaction.id(), transaction.clone()));
                 }
             }
         }
diff --git a/crates/services/relayer/src/service.rs b/crates/services/relayer/src/service.rs
index 083b6d19533..87b12b29b01 100644
--- a/crates/services/relayer/src/service.rs
+++ b/crates/services/relayer/src/service.rs
@@ -78,6 +78,8 @@ pub struct NotInitializedTask<P, D> {
     database: D,
     /// Configuration settings.
     config: Config,
+    /// Retry on error
+    retry_on_error: bool,
 }
 
 /// The actual relayer background task that syncs with the DA layer.
@@ -93,17 +95,20 @@ pub struct Task<P, D> {
     /// The watcher used to track the state of the service. If the service stops,
     /// the task will stop synchronization.
     shutdown: StateWatcher,
+    /// Retry on error
+    retry_on_error: bool,
 }
 
 impl<P, D> NotInitializedTask<P, D> {
     /// Create a new relayer task.
-    fn new(eth_node: P, database: D, config: Config) -> Self {
+    fn new(eth_node: P, database: D, config: Config, retry_on_error: bool) -> Self {
         let (synced, _) = watch::channel(None);
         Self {
             synced,
             eth_node,
             database,
             config,
+            retry_on_error,
         }
     }
 }
@@ -153,6 +158,7 @@ where
             self.config.log_page_size,
         );
         let logs = logs.take_until(self.shutdown.while_started());
+
         write_logs(&mut self.database, logs).await
     }
 
@@ -193,6 +199,7 @@ where
             eth_node,
             database,
             config,
+            retry_on_error,
         } = self;
         let mut task = Task {
             synced,
@@ -200,6 +207,7 @@ where
             database,
             config,
             shutdown,
+            retry_on_error,
         };
         task.set_deploy_height();
 
@@ -215,7 +223,6 @@ where
 {
     async fn run(&mut self, _: &mut StateWatcher) -> anyhow::Result<bool> {
         let now = tokio::time::Instant::now();
-        let should_continue = true;
 
         let result = run::run(self).await;
 
@@ -231,7 +238,18 @@ where
             .await;
         }
 
-        result.map(|_| should_continue)
+        if let Err(err) = result {
+            if !self.retry_on_error {
+                tracing::error!("Exiting due to Error in relayer task: {:?}", err);
+                let should_continue = false;
+                Ok(should_continue)
+            } else {
+                Err(err)
+            }
+        } else {
+            let should_continue = true;
+            Ok(should_continue)
+        }
     }
 
     async fn shutdown(self) -> anyhow::Result<()> {
@@ -331,7 +349,13 @@ where
     // TODO: Does this handle https?
     let http = Http::new(url);
     let eth_node = Provider::new(http);
-    Ok(new_service_internal(eth_node, database, config))
+    let retry_on_error = true;
+    Ok(new_service_internal(
+        eth_node,
+        database,
+        config,
+        retry_on_error,
+    ))
 }
 
 #[cfg(any(test, feature = "test-helpers"))]
@@ -345,19 +369,21 @@ where
     P: Middleware<Error = ProviderError> + 'static,
     D: RelayerDb + Clone + 'static,
 {
-    new_service_internal(eth_node, database, config)
+    let retry_on_fail = false;
+    new_service_internal(eth_node, database, config, retry_on_fail)
 }
 
 fn new_service_internal<P, D>(
     eth_node: P,
     database: D,
     config: Config,
+    retry_on_error: bool,
 ) -> CustomizableService<P, D>
 where
     P: Middleware<Error = ProviderError> + 'static,
     D: RelayerDb + Clone + 'static,
 {
-    let task = NotInitializedTask::new(eth_node, database, config);
+    let task = NotInitializedTask::new(eth_node, database, config, retry_on_error);
 
     CustomizableService::new(task)
 }
diff --git a/crates/services/relayer/src/service/get_logs.rs b/crates/services/relayer/src/service/get_logs.rs
index 8a7761d198f..3b92c93e6b7 100644
--- a/crates/services/relayer/src/service/get_logs.rs
+++ b/crates/services/relayer/src/service/get_logs.rs
@@ -68,24 +68,24 @@ where
     while let Some((last_height, events)) = logs.try_next().await? {
         let last_height = last_height.into();
         let mut ordered_events = BTreeMap::<DaBlockHeight, Vec<Event>>::new();
-        let fuel_events =
-            events
-                .into_iter()
-                .filter_map(|event| match EthEventLog::try_from(&event) {
-                    Ok(event) => {
-                        match event {
-                            EthEventLog::Message(m) => {
-                                Some(Ok(Event::Message(Message::from(&m))))
-                            }
-                            EthEventLog::Transaction(tx) => {
-                                Some(Ok(Event::Transaction(RelayedTransaction::from(tx))))
-                            }
-                            // TODO: Log out ignored messages.
-                            EthEventLog::Ignored => None,
+        let sorted_events = sort_events_by_log_index(events)?;
+        let fuel_events = sorted_events.into_iter().filter_map(|event| {
+            match EthEventLog::try_from(&event) {
+                Ok(event) => {
+                    match event {
+                        EthEventLog::Message(m) => {
+                            Some(Ok(Event::Message(Message::from(&m))))
                         }
+                        EthEventLog::Transaction(tx) => {
+                            Some(Ok(Event::Transaction(RelayedTransaction::from(tx))))
+                        }
+                        // TODO: Log out ignored messages.
+                        EthEventLog::Ignored => None,
                     }
-                    Err(e) => Some(Err(e)),
-                });
+                }
+                Err(e) => Some(Err(e)),
+            }
+        });
 
         for event in fuel_events {
             let event = event?;
@@ -109,3 +109,18 @@ where
     }
     Ok(())
 }
+
+fn sort_events_by_log_index(events: Vec<Log>) -> anyhow::Result<Vec<Log>> {
+    let mut with_indexes = events
+        .into_iter()
+        .map(|e| {
+            let log_index = e
+                .log_index
+                .ok_or(anyhow::anyhow!("Log missing `log_index`: {e:?}"))?;
+            Ok((log_index, e))
+        })
+        .collect::<anyhow::Result<Vec<_>>>()?;
+    with_indexes.sort_by(|(index_a, _a), (index_b, _b)| index_a.cmp(index_b));
+    let new_events = with_indexes.into_iter().map(|(_, e)| e).collect();
+    Ok(new_events)
+}
diff --git a/crates/services/src/service.rs b/crates/services/src/service.rs
index 76a0f31b000..8374c5e448f 100644
--- a/crates/services/src/service.rs
+++ b/crates/services/src/service.rs
@@ -12,6 +12,7 @@ use fuel_core_metrics::{
     },
 };
 use futures::FutureExt;
+use std::any::Any;
 use tokio::sync::watch;
 use tracing::Instrument;
 
@@ -174,7 +175,7 @@ where
         loop {
             let state = start.borrow().clone();
             if !state.starting() {
-                return Ok(state)
+                return Ok(state);
             }
             start.changed().await?;
         }
@@ -184,7 +185,7 @@ where
         loop {
             let state = stop.borrow().clone();
             if state.stopped() {
-                return Ok(state)
+                return Ok(state);
             }
             stop.changed().await?;
         }
@@ -332,7 +333,7 @@ async fn run<S>(
 
     // If the state after update is not `Starting` then return to stop the service.
     if !state.borrow().starting() {
-        return
+        return;
     }
 
     // We can panic here, because it is inside of the task.
@@ -351,6 +352,20 @@ async fn run<S>(
         }
     });
 
+    let got_panic = run_task(&mut task, state, &metric).await;
+
+    let got_panic = shutdown_task(S::NAME, task, got_panic).await;
+
+    if let Some(panic) = got_panic {
+        std::panic::resume_unwind(panic)
+    }
+}
+
+async fn run_task<S: RunnableTask>(
+    task: &mut S,
+    mut state: StateWatcher,
+    metric: &ServiceLifecycle,
+) -> Option<Box<dyn Any + Send>> {
     let mut got_panic = None;
 
     while state.borrow_and_update().started() {
@@ -361,7 +376,7 @@ async fn run<S>(
         if let Err(panic) = panic_result {
             tracing::debug!("got a panic");
             got_panic = Some(panic);
-            break
+            break;
         }
 
         let tracked_result = panic_result.expect("Checked the panic above");
@@ -382,7 +397,7 @@ async fn run<S>(
             Ok(should_continue) => {
                 if !should_continue {
                     tracing::debug!("stopping");
-                    break
+                    break;
                 }
                 tracing::debug!("run loop");
             }
@@ -392,8 +407,18 @@ async fn run<S>(
             }
         }
     }
+    got_panic
+}
 
-    tracing::info!("Shutting down {} service", S::NAME);
+async fn shutdown_task<S>(
+    name: &str,
+    task: S,
+    mut got_panic: Option<Box<dyn Any + Send>>,
+) -> Option<Box<dyn Any + Send>>
+where
+    S: RunnableTask,
+{
+    tracing::info!("Shutting down {} service", name);
     let shutdown = std::panic::AssertUnwindSafe(task.shutdown());
     match shutdown.catch_unwind().await {
         Ok(Ok(_)) => {}
@@ -412,10 +437,7 @@ async fn run<S>(
             }
         }
     }
-
-    if let Some(panic) = got_panic {
-        std::panic::resume_unwind(panic)
-    }
+    got_panic
 }
 
 impl<T> SharedMutex<T> {