Repository: quinn-rs/quinn (Rust)

Issue:
Can't send/receive more than `stream_window_bidi` roundtrip messages without the connection timing out
Hi all,

I'm currently trying to write my own QUIC server and client based on quinn v0.6.0, and I can't make more than `stream_window_bidi` roundtrip messages without the connection timing out. I have a minimal reproduction that narrows it down, but I'm unsure what is causing the problem.

Say, for example, that `stream_window_bidi` is set to 10 on the server side. The first 10 bidi streams the client opens on the connection work fine. On the 11th, the client is able to send the data, the server is able to receive the data, but the server hangs trying to write the response to the send stream, and the connection eventually times out. Enabling keepalive just makes the server hang forever. 

I've got a minimal reproduction repo available at https://github.com/SriRamanujam/quinn_minimal_network_repro. `stream_window_bidi` has been set to 10 on the server side. The client simply makes requests in a loop once a second. I've also attached traces for the client and server. 

[client_trace.log](https://github.com/djc/quinn/files/4398109/client_trace.log)
[server_trace.log](https://github.com/djc/quinn/files/4398110/server_trace.log)

/cc @Ralith 


Discussion:
Happens to me as well.
Thanks for this excellent bug report! The repro made identifying the root cause easy. I've drafted a fix at https://github.com/djc/quinn/pull/695.



Patch:
diff --git a/quinn-proto/Cargo.toml b/quinn-proto/Cargo.toml
index 96d0890edb..8250dc231e 100644
--- a/quinn-proto/Cargo.toml
+++ b/quinn-proto/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "quinn-proto"
-version = "0.6.0"
+version = "0.6.1"
 authors = ["Benjamin Saunders <ben.e.saunders@gmail.com>", "Dirkjan Ochtman <dirkjan@ochtman.nl>"]
 license = "MIT/Apache-2.0"
 repository = "https://github.com/djc/quinn"
diff --git a/quinn-proto/src/connection.rs b/quinn-proto/src/connection.rs
index c66d5c35b8..ee45fef86c 100644
--- a/quinn-proto/src/connection.rs
+++ b/quinn-proto/src/connection.rs
@@ -2723,12 +2723,8 @@ where
         if self.state.is_closed() {
             return None;
         }
-        let id = self.streams.open(self.side, dir)?;
-        // TODO: Queue STREAM_ID_BLOCKED if None
-        self.streams.send_mut(id).unwrap().max_data = match dir {
-            Dir::Uni => self.params.initial_max_stream_data_uni,
-            Dir::Bi => self.params.initial_max_stream_data_bidi_remote,
-        } as u64;
+        // TODO: Queue STREAM_ID_BLOCKED if this fails
+        let id = self.streams.open(&self.params, self.side, dir)?;
         Some(id)
     }
 
@@ -2750,7 +2746,8 @@ where
                 space.pending.max_uni_stream_id = true;
             }
         }
-        self.streams.alloc_remote_stream(self.side, dir);
+        self.streams
+            .alloc_remote_stream(&self.params, self.side, dir);
     }
 
     /// Accept a remotely initiated stream of a certain directionality, if possible
diff --git a/quinn-proto/src/streams.rs b/quinn-proto/src/streams.rs
index ded1cbcf22..667f1eb641 100644
--- a/quinn-proto/src/streams.rs
+++ b/quinn-proto/src/streams.rs
@@ -5,7 +5,8 @@ use err_derive::Error;
 use tracing::debug;
 
 use crate::{
-    assembler::Assembler, frame, range_set::RangeSet, Dir, Side, StreamId, TransportError, VarInt,
+    assembler::Assembler, frame, range_set::RangeSet, transport_parameters::TransportParameters,
+    Dir, Side, StreamId, TransportError, VarInt,
 };
 
 pub(crate) struct Streams {
@@ -40,21 +41,21 @@ impl Streams {
 
         for dir in Dir::iter() {
             for i in 0..this.max_remote[dir as usize] {
-                this.insert(true, StreamId::new(!side, dir, i));
+                this.insert(None, true, StreamId::new(!side, dir, i));
             }
         }
 
         this
     }
 
-    pub fn open(&mut self, side: Side, dir: Dir) -> Option<StreamId> {
+    pub fn open(&mut self, params: &TransportParameters, side: Side, dir: Dir) -> Option<StreamId> {
         if self.next[dir as usize] >= self.max[dir as usize] {
             return None;
         }
 
         self.next[dir as usize] += 1;
         let id = StreamId::new(side, dir, self.next[dir as usize] - 1);
-        self.insert(false, id);
+        self.insert(Some(params), false, id);
         self.send_streams += 1;
         Some(id)
     }
@@ -63,10 +64,10 @@ impl Streams {
         self.send_streams
     }
 
-    pub fn alloc_remote_stream(&mut self, side: Side, dir: Dir) {
+    pub fn alloc_remote_stream(&mut self, params: &TransportParameters, side: Side, dir: Dir) {
         self.max_remote[dir as usize] += 1;
         let id = StreamId::new(!side, dir, self.max_remote[dir as usize] - 1);
-        self.insert(true, id);
+        self.insert(Some(params), true, id);
     }
 
     pub fn accept(&mut self, side: Side, dir: Dir) -> Option<StreamId> {
@@ -201,10 +202,17 @@ impl Streams {
         id.index() >= self.next[id.dir() as usize]
     }
 
-    fn insert(&mut self, remote: bool, id: StreamId) {
+    fn insert(&mut self, params: Option<&TransportParameters>, remote: bool, id: StreamId) {
         let bi = id.dir() == Dir::Bi;
         if bi || !remote {
-            assert!(self.send.insert(id, Send::new()).is_none());
+            let max_data = params.map_or(0, |params| match id.dir() {
+                Dir::Uni => params.initial_max_stream_data_uni,
+                // Remote/local appear reversed here because the transport parameters are named from
+                // the perspective of the peer.
+                Dir::Bi if remote => params.initial_max_stream_data_bidi_local,
+                Dir::Bi => params.initial_max_stream_data_bidi_remote,
+            });
+            assert!(self.send.insert(id, Send::new(max_data)).is_none());
         }
         if bi || remote {
             assert!(self.recv.insert(id, Recv::new()).is_none());
@@ -222,10 +230,10 @@ pub(crate) struct Send {
 }
 
 impl Send {
-    pub fn new() -> Self {
+    pub fn new(max_data: u64) -> Self {
         Self {
             offset: 0,
-            max_data: 0,
+            max_data,
             state: SendState::Ready,
             bytes_in_flight: 0,
         }
diff --git a/quinn/Cargo.toml b/quinn/Cargo.toml
index 8a9049b64a..071f4c45f4 100644
--- a/quinn/Cargo.toml
+++ b/quinn/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "quinn"
-version = "0.6.0"
+version = "0.6.1"
 authors = ["Benjamin Saunders <ben.e.saunders@gmail.com>", "Dirkjan Ochtman <dirkjan@ochtman.nl>"]
 license = "MIT/Apache-2.0"
 repository = "https://github.com/djc/quinn"
@@ -32,7 +32,7 @@ err-derive = "0.2.3"
 futures = "0.3.1"
 libc = "0.2.67"
 mio = "0.6"
-proto = { package = "quinn-proto", path = "../quinn-proto", version = "0.6.0" }
+proto = { package = "quinn-proto", path = "../quinn-proto", version = "0.6.1" }
 rustls = { version = "0.17", features = ["quic"], optional = true }
 tracing = "0.1.10"
 tokio = { version = "0.2.6", features = ["rt-core", "io-driver", "time"] }


Test patch:
diff --git a/quinn-proto/src/tests/mod.rs b/quinn-proto/src/tests/mod.rs
index 1bb1a3bb2e..da8554f8af 100644
--- a/quinn-proto/src/tests/mod.rs
+++ b/quinn-proto/src/tests/mod.rs
@@ -1390,3 +1390,46 @@ fn finish_retransmit() {
     );
     assert_matches!(pair.server_conn_mut(server_ch).read_unordered(s), Ok(None));
 }
+
+/// Ensures that exchanging data on a client-initiated bidirectional stream works past the initial
+/// stream window.
+#[test]
+fn repeated_request_response() {
+    let _guard = subscribe();
+    let server = ServerConfig {
+        transport: Arc::new(TransportConfig {
+            stream_window_bidi: 1,
+            ..TransportConfig::default()
+        }),
+        ..server_config()
+    };
+    let mut pair = Pair::new(Default::default(), server);
+    let (client_ch, server_ch) = pair.connect();
+    const REQUEST: &[u8] = b"hello";
+    const RESPONSE: &[u8] = b"world";
+    for _ in 0..3 {
+        let s = pair.client_conn_mut(client_ch).open(Dir::Bi).unwrap();
+
+        pair.client_conn_mut(client_ch).write(s, REQUEST).unwrap();
+        pair.client_conn_mut(client_ch).finish(s).unwrap();
+
+        pair.drive();
+
+        assert_eq!(pair.server_conn_mut(server_ch).accept(Dir::Bi), Some(s));
+        assert_matches!(
+            pair.server_conn_mut(server_ch).read_unordered(s),
+            Ok(Some((ref data, 0))) if data == REQUEST
+        );
+        assert_matches!(pair.server_conn_mut(server_ch).read_unordered(s), Ok(None));
+        pair.server_conn_mut(server_ch).write(s, RESPONSE).unwrap();
+        pair.server_conn_mut(server_ch).finish(s).unwrap();
+
+        pair.drive();
+
+        assert_matches!(
+            pair.client_conn_mut(client_ch).read_unordered(s),
+            Ok(Some((ref data, 0))) if data == RESPONSE
+        );
+        assert_matches!(pair.client_conn_mut(client_ch).read_unordered(s), Ok(None));
+    }
+}