Repository: n0-computer/iroh
Pull request #3294: fix(iroh-relay): don't stop relay client actor if queues become full
Author: login: Frando
Created: 2025-05-08T09:08:02Z | Merged: 2025-05-08T13:46:30Z

## Description

This is a fix to the iroh-relay server.

Current situation: If a client stream becomes full (because the client is not reading fast enough from the network), the internal queue where messages to that client are queued becomes full too. Now, when another client wants to send a message to the filled-up client, instead of skipping over the message we propagate the error such that the sending client's actor dies. This is bad, we should instead just drop the message.

This PR changes the behavior to log the failed forward (at WARN level) but does not stop the sending client's actor.

It also adds a test that fails without the change and passes with the change.

## Breaking Changes

<!-- Optional, if there are any breaking changes document them, including how to migrate older code. -->

## Notes & open questions

<!-- Any notes, remarks or open questions you have to make about the PR. -->

## Change checklist
<!-- Remove any that are not relevant. -->
- [x] Self-review.
- [x] Tests if relevant.



Diff:
diff --git a/iroh-relay/src/server.rs b/iroh-relay/src/server.rs
index 3608877d531..ca8d4935cbc 100644
--- a/iroh-relay/src/server.rs
+++ b/iroh-relay/src/server.rs
@@ -1252,4 +1252,37 @@ mod tests {
 
         Ok(())
     }
+
+    #[tokio::test]
+    #[traced_test]
+    async fn test_relay_clients_full() -> TestResult<()> {
+        let server = spawn_local_relay().await.unwrap();
+        let relay_url = format!("http://{}", server.http_addr().unwrap());
+        let relay_url: RelayUrl = relay_url.parse().unwrap();
+
+        // set up client a
+        let a_secret_key = SecretKey::generate(rand::thread_rng());
+        let resolver = dns_resolver();
+        let mut client_a = ClientBuilder::new(relay_url.clone(), a_secret_key, resolver.clone())
+            .connect()
+            .await?;
+
+        // set up client b
+        let b_secret_key = SecretKey::generate(rand::thread_rng());
+        let b_key = b_secret_key.public();
+        let _client_b = ClientBuilder::new(relay_url.clone(), b_secret_key, resolver.clone())
+            .connect()
+            .await?;
+
+        // send messages from a to b, without b receiving anything.
+        // we should still keep succeeding to send, even if the packet won't be forwarded
+        // by the relay server because the server's send queue for b fills up.
+        let msg = Bytes::from("hello, b");
+        for _i in 0..1000 {
+            client_a
+                .send(SendMessage::SendPacket(b_key, msg.clone()))
+                .await?;
+        }
+        Ok(())
+    }
 }
diff --git a/iroh-relay/src/server/client.rs b/iroh-relay/src/server/client.rs
index ec6832c4243..d04eb58e1e3 100644
--- a/iroh-relay/src/server/client.rs
+++ b/iroh-relay/src/server/client.rs
@@ -365,7 +365,11 @@ impl Actor {
         match frame {
             Frame::SendPacket { dst_key, packet } => {
                 let packet_len = packet.len();
-                self.handle_frame_send_packet(dst_key, packet)?;
+                if let Err(err @ ForwardPacketError { .. }) =
+                    self.handle_frame_send_packet(dst_key, packet)
+                {
+                    warn!("failed to handle send packet frame: {err:#}");
+                }
                 self.metrics.bytes_recv.inc_by(packet_len as u64);
             }
             Frame::Ping { data } => {
@@ -387,7 +391,7 @@ impl Actor {
         Ok(())
     }
 
-    fn handle_frame_send_packet(&self, dst: NodeId, data: Bytes) -> Result<()> {
+    fn handle_frame_send_packet(&self, dst: NodeId, data: Bytes) -> Result<(), ForwardPacketError> {
         if disco::looks_like_disco_wrapper(&data) {
             self.metrics.disco_packets_recv.inc();
             self.clients
@@ -401,6 +405,31 @@ impl Actor {
     }
 }
 
+#[derive(Debug)]
+pub(crate) enum PacketScope {
+    Disco,
+    Data,
+}
+
+#[derive(Debug)]
+pub(crate) enum SendError {
+    Full,
+    Closed,
+}
+
+#[derive(Debug, thiserror::Error)]
+#[error("failed to forward {scope:?} packet: {reason:?}")]
+pub(crate) struct ForwardPacketError {
+    scope: PacketScope,
+    reason: SendError,
+}
+
+impl ForwardPacketError {
+    pub(crate) fn new(scope: PacketScope, reason: SendError) -> Self {
+        Self { scope, reason }
+    }
+}
+
 /// Rate limiter for reading from a [`RelayedStream`].
 ///
 /// The writes to the sink are not rate limited.
diff --git a/iroh-relay/src/server/clients.rs b/iroh-relay/src/server/clients.rs
index ed71e93dd7b..19b6cb8c842 100644
--- a/iroh-relay/src/server/clients.rs
+++ b/iroh-relay/src/server/clients.rs
@@ -9,15 +9,18 @@ use std::{
     },
 };
 
-use anyhow::{bail, Result};
+use anyhow::Result;
 use bytes::Bytes;
 use dashmap::DashMap;
 use iroh_base::NodeId;
 use tokio::sync::mpsc::error::TrySendError;
 use tracing::{debug, trace};
 
-use super::client::{Client, Config};
-use crate::server::metrics::Metrics;
+use super::client::{Client, Config, ForwardPacketError};
+use crate::server::{
+    client::{PacketScope, SendError},
+    metrics::Metrics,
+};
 
 /// Manages the connections to all currently connected clients.
 #[derive(Debug, Default, Clone)]
@@ -109,7 +112,7 @@ impl Clients {
         data: Bytes,
         src: NodeId,
         metrics: &Metrics,
-    ) -> Result<()> {
+    ) -> Result<(), ForwardPacketError> {
         let Some(client) = self.0.clients.get(&dst) else {
             debug!(dst = dst.fmt_short(), "no connected client, dropped packet");
             metrics.send_packets_dropped.inc();
@@ -126,7 +129,7 @@ impl Clients {
                     dst = dst.fmt_short(),
                     "client too busy to receive packet, dropping packet"
                 );
-                bail!("failed to send message: full");
+                Err(ForwardPacketError::new(PacketScope::Data, SendError::Full))
             }
             Err(TrySendError::Closed(_)) => {
                 debug!(
@@ -134,7 +137,10 @@ impl Clients {
                     "can no longer write to client, dropping message and pruning connection"
                 );
                 client.start_shutdown();
-                bail!("failed to send message: gone");
+                Err(ForwardPacketError::new(
+                    PacketScope::Data,
+                    SendError::Closed,
+                ))
             }
         }
     }
@@ -146,7 +152,7 @@ impl Clients {
         data: Bytes,
         src: NodeId,
         metrics: &Metrics,
-    ) -> Result<()> {
+    ) -> Result<(), ForwardPacketError> {
         let Some(client) = self.0.clients.get(&dst) else {
             debug!(
                 dst = dst.fmt_short(),
@@ -166,7 +172,7 @@ impl Clients {
                     dst = dst.fmt_short(),
                     "client too busy to receive disco packet, dropping packet"
                 );
-                bail!("failed to send message: full");
+                Err(ForwardPacketError::new(PacketScope::Disco, SendError::Full))
             }
             Err(TrySendError::Closed(_)) => {
                 debug!(
@@ -174,7 +180,10 @@ impl Clients {
                     "can no longer write to client, dropping disco message and pruning connection"
                 );
                 client.start_shutdown();
-                bail!("failed to send message: gone");
+                Err(ForwardPacketError::new(
+                    PacketScope::Disco,
+                    SendError::Closed,
+                ))
             }
         }
     }


Reviews:
user: login: flub
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-08T09:19:25Z
user: login: Frando
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-08T09:49:24Z
user: login: Frando
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-08T09:50:26Z
user: login: Frando
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-08T09:51:05Z
user: login: flub
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-05-08T09:55:19Z
user: login: flub
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-08T10:03:34Z
user: login: Frando
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-08T11:23:27Z
user: login: flub
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-05-08T11:32:43Z
user: login: flub
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-08T11:33:02Z

Review comments:
diff_hunk: @@ -166,15 +166,15 @@ impl Clients {
                     dst = dst.fmt_short(),
                     "client too busy to receive disco packet, dropping packet"
                 );
-                bail!("failed to send message: full");
+                bail!("failed to send disco message: full");
path: iroh-relay/src/server/clients.rs
user: login: flub
body: Thanks for disambiguating these!  Did you trace these calls up to see how the error is handled as well, to make sure it doesn't contain a similar bug as the other send path?
created_at: 2025-05-08T09:15:28Z
author_association: CONTRIBUTOR
diff_hunk: @@ -365,7 +365,9 @@ impl Actor {
         match frame {
             Frame::SendPacket { dst_key, packet } => {
                 let packet_len = packet.len();
-                self.handle_frame_send_packet(dst_key, packet)?;
+                if let Err(err) = self.handle_frame_send_packet(dst_key, packet) {
+                    warn!("failed to handle send packet frame: {err:#}");
path: iroh-relay/src/server/client.rs
user: login: flub
body: Did you check if it's fine to ignore *all* errors here?  What other errors can occur?
created_at: 2025-05-08T09:16:12Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1252,4 +1252,37 @@ mod tests {
 
         Ok(())
     }
+
+    #[tokio::test]
+    #[traced_test]
+    async fn test_relay_clients_full() -> TestResult<()> {
+        let server = spawn_local_relay().await.unwrap();
+        let relay_url = format!("http://{}", server.http_addr().unwrap());
+        let relay_url: RelayUrl = relay_url.parse().unwrap();
+
+        // set up client a
+        let a_secret_key = SecretKey::generate(rand::thread_rng());
+        let resolver = dns_resolver();
+        let mut client_a = ClientBuilder::new(relay_url.clone(), a_secret_key, resolver.clone())
+            .connect()
+            .await?;
+
+        // set up client b
+        let b_secret_key = SecretKey::generate(rand::thread_rng());
+        let b_key = b_secret_key.public();
+        let _client_b = ClientBuilder::new(relay_url.clone(), b_secret_key, resolver.clone())
+            .connect()
+            .await?;
+
+        // send messages from a to b, without b receiving anything.
+        // we should still keep succeeding to send, even if the packet won't be forwarded
+        // by the relay server because the server's send queue for b fills up.
path: iroh-relay/src/server.rs
user: login: flub
body: Do you have an estimate of the number of messages that get queued?  Or did you just try something random?  Which is also fine, I'm just curious.
created_at: 2025-05-08T09:19:20Z
author_association: CONTRIBUTOR
diff_hunk: @@ -166,15 +166,15 @@ impl Clients {
                     dst = dst.fmt_short(),
                     "client too busy to receive disco packet, dropping packet"
                 );
-                bail!("failed to send message: full");
+                bail!("failed to send disco message: full");
path: iroh-relay/src/server/clients.rs
user: login: Frando
body: I did, it is the same call path from one layer up on.
created_at: 2025-05-08T09:49:24Z
author_association: MEMBER
diff_hunk: @@ -365,7 +365,9 @@ impl Actor {
         match frame {
             Frame::SendPacket { dst_key, packet } => {
                 let packet_len = packet.len();
-                self.handle_frame_send_packet(dst_key, packet)?;
+                if let Err(err) = self.handle_frame_send_packet(dst_key, packet) {
+                    warn!("failed to handle send packet frame: {err:#}");
path: iroh-relay/src/server/client.rs
user: login: Frando
body: Yes, the channel being full or gone is the only error. I just pushed a commit that adds a proper error type to make sure that this invariant holds.
created_at: 2025-05-08T09:50:26Z
author_association: MEMBER
diff_hunk: @@ -1252,4 +1252,37 @@ mod tests {
 
         Ok(())
     }
+
+    #[tokio::test]
+    #[traced_test]
+    async fn test_relay_clients_full() -> TestResult<()> {
+        let server = spawn_local_relay().await.unwrap();
+        let relay_url = format!("http://{}", server.http_addr().unwrap());
+        let relay_url: RelayUrl = relay_url.parse().unwrap();
+
+        // set up client a
+        let a_secret_key = SecretKey::generate(rand::thread_rng());
+        let resolver = dns_resolver();
+        let mut client_a = ClientBuilder::new(relay_url.clone(), a_secret_key, resolver.clone())
+            .connect()
+            .await?;
+
+        // set up client b
+        let b_secret_key = SecretKey::generate(rand::thread_rng());
+        let b_key = b_secret_key.public();
+        let _client_b = ClientBuilder::new(relay_url.clone(), b_secret_key, resolver.clone())
+            .connect()
+            .await?;
+
+        // send messages from a to b, without b receiving anything.
+        // we should still keep succeeding to send, even if the packet won't be forwarded
+        // by the relay server because the server's send queue for b fills up.
path: iroh-relay/src/server.rs
user: login: Frando
body: With 500 it passed, with 1000 it failed.
created_at: 2025-05-08T09:51:05Z
author_association: MEMBER
diff_hunk: @@ -365,7 +365,9 @@ impl Actor {
         match frame {
             Frame::SendPacket { dst_key, packet } => {
                 let packet_len = packet.len();
-                self.handle_frame_send_packet(dst_key, packet)?;
+                if let Err(err) = self.handle_frame_send_packet(dst_key, packet) {
path: iroh-relay/src/server/client.rs
user: login: flub
body: Hum, I guess you somehow have to match the type here for your custom type to add protection against the type changing?  The `@` syntax in match might be able to do that, not sure if you can use that in `if let`.
created_at: 2025-05-08T10:03:34Z
author_association: CONTRIBUTOR
diff_hunk: @@ -365,7 +365,9 @@ impl Actor {
         match frame {
             Frame::SendPacket { dst_key, packet } => {
                 let packet_len = packet.len();
-                self.handle_frame_send_packet(dst_key, packet)?;
+                if let Err(err) = self.handle_frame_send_packet(dst_key, packet) {
path: iroh-relay/src/server/client.rs
user: login: Frando
body: Good point. This seems to work, cool:
```rust
                if let Err(err @ ForwardPacketError { .. }) =
                    self.handle_frame_send_packet(dst_key, packet)
                {
                    warn!("failed to handle send packet frame: {err:#}");
                }

```
created_at: 2025-05-08T11:23:27Z
author_association: MEMBER
diff_hunk: @@ -365,7 +365,9 @@ impl Actor {
         match frame {
             Frame::SendPacket { dst_key, packet } => {
                 let packet_len = packet.len();
-                self.handle_frame_send_packet(dst_key, packet)?;
+                if let Err(err) = self.handle_frame_send_packet(dst_key, packet) {
path: iroh-relay/src/server/client.rs
user: login: flub
body: Nice!  Good to know if-let also allows the @ syntax.
created_at: 2025-05-08T11:33:02Z
author_association: CONTRIBUTOR

PR comments:
user: login: github-actions[bot]
created_at: 2025-05-08T09:10:20Z
body: Documentation for this PR has been generated and is available at: https://n0-computer.github.io/iroh/pr/3294/docs/iroh/

Last updated: 2025-05-08T11:24:47Z
author_association: NONE
user: login: github-actions[bot]
created_at: 2025-05-08T09:16:34Z
body: Netsim report & logs for this PR have been generated and is available at: [LOGS](https://github.com/n0-computer/iroh/actions/runs/14905313471/artifacts/3085707331)
This report will remain available for 3 days.

Last updated for commit: e7f8651
author_association: NONE