Repository: quinn-rs/quinn (Rust)

Issue:
0-RTT packets can be lost due to race condition introduced along with Incoming
Unfortunately it looks like in #1752 I accidentally introduced a bug. Apologies for not catching it.

Bug description: If a client sends a 0-RTT datagram to the server after sending the datagram with its Initial packet, and it is received and processed by the endpoint before the corresponding `Incoming` is `accept`-ed, the 0-RTT packet will not find any associated connection and thus be dropped. As a consequence, this seems to cause request/response pairs on connections where this occurs to actually take 3 round trips to go through successfully.

I am working on a fix, and I will see if I can get a PR opened for it tomorrow.


Discussion:
Thanks for following up on this!



Patch:
diff --git a/quinn-proto/src/config.rs b/quinn-proto/src/config.rs
index e46416f6c8..a264342b14 100644
--- a/quinn-proto/src/config.rs
+++ b/quinn-proto/src/config.rs
@@ -776,6 +776,10 @@ pub struct ServerConfig {
 
     pub(crate) preferred_address_v4: Option<SocketAddrV4>,
     pub(crate) preferred_address_v6: Option<SocketAddrV6>,
+
+    pub(crate) max_incoming: usize,
+    pub(crate) incoming_buffer_size: u64,
+    pub(crate) incoming_buffer_size_total: u64,
 }
 
 impl ServerConfig {
@@ -795,6 +799,10 @@ impl ServerConfig {
 
             preferred_address_v4: None,
             preferred_address_v6: None,
+
+            max_incoming: 1 << 16,
+            incoming_buffer_size: 10 << 20,
+            incoming_buffer_size_total: 100 << 20,
         }
     }
 
@@ -838,6 +846,54 @@ impl ServerConfig {
         self.preferred_address_v6 = address;
         self
     }
+
+    /// Maximum number of [`Incoming`][crate::Incoming] to allow to exist at a time
+    ///
+    /// An [`Incoming`][crate::Incoming] comes into existence when an incoming connection attempt
+    /// is received and stops existing when the application either accepts it or otherwise disposes
+    /// of it. While this limit is reached, new incoming connection attempts are immediately
+    /// refused. Larger values have greater worst-case memory consumption, but accommodate greater
+    /// application latency in handling incoming connection attempts.
+    ///
+    /// The default value is set to 65536. With a typical Ethernet MTU of 1500 bytes, this limits
+    /// memory consumption from this to under 100 MiB--a generous amount that still prevents memory
+    /// exhaustion in most contexts.
+    pub fn max_incoming(&mut self, max_incoming: usize) -> &mut Self {
+        self.max_incoming = max_incoming;
+        self
+    }
+
+    /// Maximum number of received bytes to buffer for each [`Incoming`][crate::Incoming]
+    ///
+    /// An [`Incoming`][crate::Incoming] comes into existence when an incoming connection attempt
+    /// is received and stops existing when the application either accepts it or otherwise disposes
+    /// of it. This limit governs only packets received within that period, and does not include
+    /// the first packet. Packets received in excess of this limit are dropped, which may cause
+    /// 0-RTT or handshake data to have to be retransmitted.
+    ///
+    /// The default value is set to 10 MiB--an amount such that in most situations a client would
+    /// not transmit that much 0-RTT data faster than the server handles the corresponding
+    /// [`Incoming`][crate::Incoming].
+    pub fn incoming_buffer_size(&mut self, incoming_buffer_size: u64) -> &mut Self {
+        self.incoming_buffer_size = incoming_buffer_size;
+        self
+    }
+
+    /// Maximum number of received bytes to buffer for all [`Incoming`][crate::Incoming]
+    /// collectively
+    ///
+    /// An [`Incoming`][crate::Incoming] comes into existence when an incoming connection attempt
+    /// is received and stops existing when the application either accepts it or otherwise disposes
+    /// of it. This limit governs only packets received within that period, and does not include
+    /// the first packet. Packets received in excess of this limit are dropped, which may cause
+    /// 0-RTT or handshake data to have to be retransmitted.
+    ///
+    /// The default value is set to 100 MiB--a generous amount that still prevents memory
+    /// exhaustion in most contexts.
+    pub fn incoming_buffer_size_total(&mut self, incoming_buffer_size_total: u64) -> &mut Self {
+        self.incoming_buffer_size_total = incoming_buffer_size_total;
+        self
+    }
 }
 
 #[cfg(feature = "rustls")]
@@ -880,6 +936,12 @@ impl fmt::Debug for ServerConfig {
             .field("migration", &self.migration)
             .field("preferred_address_v4", &self.preferred_address_v4)
             .field("preferred_address_v6", &self.preferred_address_v6)
+            .field("max_incoming", &self.max_incoming)
+            .field("incoming_buffer_size", &self.incoming_buffer_size)
+            .field(
+                "incoming_buffer_size_total",
+                &self.incoming_buffer_size_total,
+            )
             .finish()
     }
 }
diff --git a/quinn-proto/src/connection/mod.rs b/quinn-proto/src/connection/mod.rs
index 8a067302c9..51ca2855ae 100644
--- a/quinn-proto/src/connection/mod.rs
+++ b/quinn-proto/src/connection/mod.rs
@@ -25,8 +25,8 @@ use crate::{
     packet::{Header, InitialHeader, InitialPacket, LongType, Packet, PartialDecode, SpaceId},
     range_set::ArrayRangeSet,
     shared::{
-        ConnectionEvent, ConnectionEventInner, ConnectionId, EcnCodepoint, EndpointEvent,
-        EndpointEventInner,
+        ConnectionEvent, ConnectionEventInner, ConnectionId, DatagramConnectionEvent, EcnCodepoint,
+        EndpointEvent, EndpointEventInner,
     },
     token::ResetToken,
     transport_parameters::TransportParameters,
@@ -989,13 +989,13 @@ impl Connection {
     pub fn handle_event(&mut self, event: ConnectionEvent) {
         use self::ConnectionEventInner::*;
         match event.0 {
-            Datagram {
+            Datagram(DatagramConnectionEvent {
                 now,
                 remote,
                 ecn,
                 first_decode,
                 remaining,
-            } => {
+            }) => {
                 // If this packet could initiate a migration and we're a client or a server that
                 // forbids migration, drop the datagram. This could be relaxed to heuristically
                 // permit NAT-rebinding-like migration.
@@ -3346,11 +3346,11 @@ impl Connection {
     #[cfg(test)]
     pub(crate) fn decode_packet(&self, event: &ConnectionEvent) -> Option<Vec<u8>> {
         let (first_decode, remaining) = match &event.0 {
-            ConnectionEventInner::Datagram {
+            ConnectionEventInner::Datagram(DatagramConnectionEvent {
                 first_decode,
                 remaining,
                 ..
-            } => (first_decode, remaining),
+            }) => (first_decode, remaining),
             _ => return None,
         };
 
diff --git a/quinn-proto/src/endpoint.rs b/quinn-proto/src/endpoint.rs
index 898cb4368b..3bfab440f8 100644
--- a/quinn-proto/src/endpoint.rs
+++ b/quinn-proto/src/endpoint.rs
@@ -1,7 +1,7 @@
 use std::{
     collections::{hash_map, HashMap},
     convert::TryFrom,
-    fmt,
+    fmt, mem,
     net::{IpAddr, SocketAddr},
     ops::{Index, IndexMut},
     sync::Arc,
@@ -27,8 +27,8 @@ use crate::{
         PartialDecode, PlainInitialHeader,
     },
     shared::{
-        ConnectionEvent, ConnectionEventInner, ConnectionId, EcnCodepoint, EndpointEvent,
-        EndpointEventInner, IssuedCid,
+        ConnectionEvent, ConnectionEventInner, ConnectionId, DatagramConnectionEvent, EcnCodepoint,
+        EndpointEvent, EndpointEventInner, IssuedCid,
     },
     token::TokenDecodeError,
     transport_parameters::{PreferredAddress, TransportParameters},
@@ -51,6 +51,9 @@ pub struct Endpoint {
     allow_mtud: bool,
     /// Time at which a stateless reset was most recently sent
     last_stateless_reset: Option<Instant>,
+    /// Buffered Initial and 0-RTT messages for pending incoming connections
+    incoming_buffers: Slab<IncomingBuffer>,
+    all_incoming_buffers_total_bytes: u64,
 }
 
 impl Endpoint {
@@ -74,6 +77,8 @@ impl Endpoint {
             server_config,
             allow_mtud,
             last_stateless_reset: None,
+            incoming_buffers: Slab::new(),
+            all_incoming_buffers_total_bytes: 0,
         }
     }
 
@@ -189,17 +194,42 @@ impl Endpoint {
         //
 
         let addresses = FourTuple { remote, local_ip };
-        if let Some(ch) = self.index.get(&addresses, &first_decode) {
-            return Some(DatagramEvent::ConnectionEvent(
-                ch,
-                ConnectionEvent(ConnectionEventInner::Datagram {
-                    now,
-                    remote: addresses.remote,
-                    ecn,
-                    first_decode,
-                    remaining,
-                }),
-            ));
+        if let Some(route_to) = self.index.get(&addresses, &first_decode) {
+            let event = DatagramConnectionEvent {
+                now,
+                remote: addresses.remote,
+                ecn,
+                first_decode,
+                remaining,
+            };
+            match route_to {
+                RouteDatagramTo::Incoming(incoming_idx) => {
+                    let incoming_buffer = &mut self.incoming_buffers[incoming_idx];
+                    let config = &self.server_config.as_ref().unwrap();
+
+                    if incoming_buffer
+                        .total_bytes
+                        .checked_add(datagram_len as u64)
+                        .map_or(false, |n| n <= config.incoming_buffer_size)
+                        && self
+                            .all_incoming_buffers_total_bytes
+                            .checked_add(datagram_len as u64)
+                            .map_or(false, |n| n <= config.incoming_buffer_size_total)
+                    {
+                        incoming_buffer.datagrams.push(event);
+                        incoming_buffer.total_bytes += datagram_len as u64;
+                        self.all_incoming_buffers_total_bytes += datagram_len as u64;
+                    }
+
+                    return None;
+                }
+                RouteDatagramTo::Connection(ch) => {
+                    return Some(DatagramEvent::ConnectionEvent(
+                        ch,
+                        ConnectionEvent(ConnectionEventInner::Datagram(event)),
+                    ))
+                }
+            }
         }
 
         //
@@ -489,6 +519,10 @@ impl Endpoint {
             }
         };
 
+        let incoming_idx = self.incoming_buffers.insert(IncomingBuffer::default());
+        self.index
+            .insert_initial_incoming(orig_dst_cid, incoming_idx);
+
         Some(DatagramEvent::NewConnection(Incoming {
             addresses,
             ecn,
@@ -501,6 +535,8 @@ impl Endpoint {
             crypto,
             retry_src_cid,
             orig_dst_cid,
+            incoming_idx,
+            improper_drop_warner: IncomingImproperDropWarner,
         }))
     }
 
@@ -512,6 +548,11 @@ impl Endpoint {
         buf: &mut Vec<u8>,
         server_config: Option<Arc<ServerConfig>>,
     ) -> Result<(ConnectionHandle, Connection), AcceptError> {
+        let remote_address_validated = incoming.remote_address_validated();
+        incoming.improper_drop_warner.dismiss();
+        let incoming_buffer = self.incoming_buffers.remove(incoming.incoming_idx);
+        self.all_incoming_buffers_total_bytes -= incoming_buffer.total_bytes;
+
         let packet_number = incoming.packet.header.number.expand(0);
         let InitialHeader {
             src_cid,
@@ -522,6 +563,7 @@ impl Endpoint {
 
         if self.cids_exhausted() {
             debug!("refusing connection");
+            self.index.remove_initial(incoming.orig_dst_cid);
             return Err(AcceptError {
                 cause: ConnectionError::CidsExhausted,
                 response: Some(self.initial_close(
@@ -550,6 +592,7 @@ impl Endpoint {
             .is_err()
         {
             debug!(packet_number, "failed to authenticate initial packet");
+            self.index.remove_initial(incoming.orig_dst_cid);
             return Err(AcceptError {
                 cause: TransportError::PROTOCOL_VIOLATION("authentication failed").into(),
                 response: None,
@@ -596,11 +639,12 @@ impl Endpoint {
             tls,
             Some(server_config),
             transport_config,
-            incoming.remote_address_validated(),
+            remote_address_validated,
         );
         if dst_cid.len() != 0 {
             self.index.insert_initial(dst_cid, ch);
         }
+
         match conn.handle_first_packet(
             now,
             incoming.addresses.remote,
@@ -611,6 +655,11 @@ impl Endpoint {
         ) {
             Ok(()) => {
                 trace!(id = ch.0, icid = %dst_cid, "new connection");
+
+                for event in incoming_buffer.datagrams {
+                    conn.handle_event(ConnectionEvent(ConnectionEventInner::Datagram(event)))
+                }
+
                 Ok((ch, conn))
             }
             Err(e) => {
@@ -637,7 +686,8 @@ impl Endpoint {
         &mut self,
         header: &PlainInitialHeader,
     ) -> Result<(), TransportError> {
-        if self.cids_exhausted() {
+        let config = &self.server_config.as_ref().unwrap();
+        if self.cids_exhausted() || self.incoming_buffers.len() >= config.max_incoming {
             return Err(TransportError::CONNECTION_REFUSED(""));
         }
 
@@ -663,6 +713,9 @@ impl Endpoint {
 
     /// Reject this incoming connection attempt
     pub fn refuse(&mut self, incoming: Incoming, buf: &mut Vec<u8>) -> Transmit {
+        self.clean_up_incoming(&incoming);
+        incoming.improper_drop_warner.dismiss();
+
         self.initial_close(
             incoming.packet.header.version,
             incoming.addresses,
@@ -680,6 +733,10 @@ impl Endpoint {
         if incoming.remote_address_validated() {
             return Err(RetryError(incoming));
         }
+
+        self.clean_up_incoming(&incoming);
+        incoming.improper_drop_warner.dismiss();
+
         let server_config = self.server_config.as_ref().unwrap();
 
         // First Initial
@@ -724,6 +781,22 @@ impl Endpoint {
         })
     }
 
+    /// Ignore this incoming connection attempt, not sending any packet in response
+    ///
+    /// Doing this actively, rather than merely dropping the [`Incoming`], is necessary to prevent
+    /// memory leaks due to state within [`Endpoint`] tracking the incoming connection.
+    pub fn ignore(&mut self, incoming: Incoming) {
+        self.clean_up_incoming(&incoming);
+        incoming.improper_drop_warner.dismiss();
+    }
+
+    /// Clean up endpoint data structures associated with an `Incoming`.
+    fn clean_up_incoming(&mut self, incoming: &Incoming) {
+        self.index.remove_initial(incoming.orig_dst_cid);
+        let incoming_buffer = self.incoming_buffers.remove(incoming.incoming_idx);
+        self.all_incoming_buffers_total_bytes -= incoming_buffer.total_bytes;
+    }
+
     fn add_connection(
         &mut self,
         ch: ConnectionHandle,
@@ -870,17 +943,37 @@ impl fmt::Debug for Endpoint {
             .field("connections", &self.connections)
             .field("config", &self.config)
             .field("server_config", &self.server_config)
+            // incoming_buffers too large
+            .field("incoming_buffers.len", &self.incoming_buffers.len())
+            .field(
+                "all_incoming_buffers_total_bytes",
+                &self.all_incoming_buffers_total_bytes,
+            )
             .finish()
     }
 }
 
+/// Buffered Initial and 0-RTT messages for a pending incoming connection
+#[derive(Default)]
+struct IncomingBuffer {
+    datagrams: Vec<DatagramConnectionEvent>,
+    total_bytes: u64,
+}
+
+/// Part of protocol state incoming datagrams can be routed to
+#[derive(Copy, Clone, Debug)]
+enum RouteDatagramTo {
+    Incoming(usize),
+    Connection(ConnectionHandle),
+}
+
 /// Maps packets to existing connections
 #[derive(Default, Debug)]
 struct ConnectionIndex {
     /// Identifies connections based on the initial DCID the peer utilized
     ///
     /// Uses a standard `HashMap` to protect against hash collision attacks.
-    connection_ids_initial: HashMap<ConnectionId, ConnectionHandle>,
+    connection_ids_initial: HashMap<ConnectionId, RouteDatagramTo>,
     /// Identifies connections based on locally created CIDs
     ///
     /// Uses a cheaper hash function since keys are locally created
@@ -897,9 +990,21 @@ struct ConnectionIndex {
 }
 
 impl ConnectionIndex {
+    /// Associate an incoming connection with its initial destination CID
+    fn insert_initial_incoming(&mut self, dst_cid: ConnectionId, incoming_key: usize) {
+        self.connection_ids_initial
+            .insert(dst_cid, RouteDatagramTo::Incoming(incoming_key));
+    }
+
+    /// Remove an association with an initial destination CID
+    fn remove_initial(&mut self, dst_cid: ConnectionId) {
+        self.connection_ids_initial.remove(&dst_cid);
+    }
+
     /// Associate a connection with its initial destination CID
     fn insert_initial(&mut self, dst_cid: ConnectionId, connection: ConnectionHandle) {
-        self.connection_ids_initial.insert(dst_cid, connection);
+        self.connection_ids_initial
+            .insert(dst_cid, RouteDatagramTo::Connection(connection));
     }
 
     /// Associate a connection with its first locally-chosen destination CID if used, or otherwise
@@ -940,10 +1045,10 @@ impl ConnectionIndex {
     }
 
     /// Find the existing connection that `datagram` should be routed to, if any
-    fn get(&self, addresses: &FourTuple, datagram: &PartialDecode) -> Option<ConnectionHandle> {
+    fn get(&self, addresses: &FourTuple, datagram: &PartialDecode) -> Option<RouteDatagramTo> {
         if datagram.dst_cid().len() != 0 {
             if let Some(&ch) = self.connection_ids.get(datagram.dst_cid()) {
-                return Some(ch);
+                return Some(RouteDatagramTo::Connection(ch));
             }
         }
         if datagram.is_initial() || datagram.is_0rtt() {
@@ -953,7 +1058,7 @@ impl ConnectionIndex {
         }
         if datagram.dst_cid().len() == 0 {
             if let Some(&ch) = self.connection_remotes.get(addresses) {
-                return Some(ch);
+                return Some(RouteDatagramTo::Connection(ch));
             }
         }
         let data = datagram.data();
@@ -963,6 +1068,7 @@ impl ConnectionIndex {
         self.connection_reset_tokens
             .get(addresses.remote, &data[data.len() - RESET_TOKEN_SIZE..])
             .cloned()
+            .map(RouteDatagramTo::Connection)
     }
 }
 
@@ -1025,6 +1131,8 @@ pub struct Incoming {
     crypto: Keys,
     retry_src_cid: Option<ConnectionId>,
     orig_dst_cid: ConnectionId,
+    incoming_idx: usize,
+    improper_drop_warner: IncomingImproperDropWarner,
 }
 
 impl Incoming {
@@ -1059,10 +1167,27 @@ impl fmt::Debug for Incoming {
             // rest is too big and not meaningful enough
             .field("retry_src_cid", &self.retry_src_cid)
             .field("orig_dst_cid", &self.orig_dst_cid)
+            .field("incoming_idx", &self.incoming_idx)
+            // improper drop warner contains no information
             .finish_non_exhaustive()
     }
 }
 
+struct IncomingImproperDropWarner;
+
+impl IncomingImproperDropWarner {
+    fn dismiss(self) {
+        mem::forget(self);
+    }
+}
+
+impl Drop for IncomingImproperDropWarner {
+    fn drop(&mut self) {
+        warn!("quinn_proto::Incoming dropped without passing to Endpoint::accept/refuse/retry/ignore \
+               (may cause memory leak and eventual inability to accept new connections)");
+    }
+}
+
 /// Errors in the parameters being used to create a new connection
 ///
 /// These arise before any I/O has been performed.
diff --git a/quinn-proto/src/shared.rs b/quinn-proto/src/shared.rs
index a8bd274b28..ed40f33b02 100644
--- a/quinn-proto/src/shared.rs
+++ b/quinn-proto/src/shared.rs
@@ -11,17 +11,21 @@ pub struct ConnectionEvent(pub(crate) ConnectionEventInner);
 #[derive(Debug)]
 pub(crate) enum ConnectionEventInner {
     /// A datagram has been received for the Connection
-    Datagram {
-        now: Instant,
-        remote: SocketAddr,
-        ecn: Option<EcnCodepoint>,
-        first_decode: PartialDecode,
-        remaining: Option<BytesMut>,
-    },
+    Datagram(DatagramConnectionEvent),
     /// New connection identifiers have been issued for the Connection
     NewIdentifiers(Vec<IssuedCid>, Instant),
 }
 
+/// Variant of [`ConnectionEventInner`].
+#[derive(Debug)]
+pub(crate) struct DatagramConnectionEvent {
+    pub(crate) now: Instant,
+    pub(crate) remote: SocketAddr,
+    pub(crate) ecn: Option<EcnCodepoint>,
+    pub(crate) first_decode: PartialDecode,
+    pub(crate) remaining: Option<BytesMut>,
+}
+
 /// Events sent from a Connection to an Endpoint
 #[derive(Debug)]
 pub struct EndpointEvent(pub(crate) EndpointEventInner);
diff --git a/quinn/src/endpoint.rs b/quinn/src/endpoint.rs
index ea5c9a1b12..f6dd29b8c8 100644
--- a/quinn/src/endpoint.rs
+++ b/quinn/src/endpoint.rs
@@ -29,7 +29,7 @@ use udp::{RecvMeta, BATCH_SIZE};
 
 use crate::{
     connection::Connecting, incoming::Incoming, work_limiter::WorkLimiter, ConnectionEvent,
-    EndpointConfig, VarInt, IO_LOOP_BOUND, MAX_INCOMING_CONNECTIONS, RECV_TIME_BOUND,
+    EndpointConfig, VarInt, IO_LOOP_BOUND, RECV_TIME_BOUND,
 };
 
 /// A QUIC endpoint.
@@ -410,6 +410,10 @@ impl EndpointInner {
         respond(transmit, &response_buffer, &*state.socket);
         Ok(())
     }
+
+    pub(crate) fn ignore(&self, incoming: proto::Incoming) {
+        self.state.lock().unwrap().inner.ignore(incoming);
+    }
 }
 
 #[derive(Debug)]
@@ -741,9 +745,7 @@ impl RecvState {
                                 &mut response_buffer,
                             ) {
                                 Some(DatagramEvent::NewConnection(incoming)) => {
-                                    if self.incoming.len() < MAX_INCOMING_CONNECTIONS
-                                        && self.connections.close.is_none()
-                                    {
+                                    if self.connections.close.is_none() {
                                         self.incoming.push_back(incoming);
                                     } else {
                                         let transmit =
diff --git a/quinn/src/incoming.rs b/quinn/src/incoming.rs
index c6a03bc170..d776874b03 100644
--- a/quinn/src/incoming.rs
+++ b/quinn/src/incoming.rs
@@ -63,7 +63,8 @@ impl Incoming {
 
     /// Ignore this incoming connection attempt, not sending any packet in response
     pub fn ignore(mut self) {
-        self.0.take().unwrap();
+        let state = self.0.take().unwrap();
+        state.endpoint.ignore(state.inner);
     }
 
     /// The local IP address which was used when the peer established
diff --git a/quinn/src/lib.rs b/quinn/src/lib.rs
index b933e8fc04..f928d9ca39 100644
--- a/quinn/src/lib.rs
+++ b/quinn/src/lib.rs
@@ -130,10 +130,3 @@ const IO_LOOP_BOUND: usize = 160;
 /// Going much lower does not yield any noticeable difference, since a single `recvmmsg`
 /// batch of size 32 was observed to take 30us on some systems.
 const RECV_TIME_BOUND: Duration = Duration::from_micros(50);
-
-/// The maximum number of `IncomingConnection`s we allow to be enqueued at a time before we start
-/// rejecting new `IncomingConnection`s automatically. Assuming each `IncomingConnection` accounts
-/// for little over 1200 bytes of memory maximum, this should limit an endpoint's incoming
-/// connection queue memory consumption to under 100 MiB, a generous amount that still prevents
-/// memory exhaustion.
-const MAX_INCOMING_CONNECTIONS: usize = 1 << 16;


Test patch:
diff --git a/quinn-proto/src/tests/mod.rs b/quinn-proto/src/tests/mod.rs
index 6bc5dbfaa0..9752d5c6b2 100644
--- a/quinn-proto/src/tests/mod.rs
+++ b/quinn-proto/src/tests/mod.rs
@@ -666,6 +666,113 @@ fn zero_rtt_rejection() {
     assert_eq!(pair.client_conn_mut(client_ch).lost_packets(), 0);
 }
 
+fn test_zero_rtt_incoming_limit<F: FnOnce(&mut ServerConfig)>(configure_server: F) {
+    // caller sets the server limit to 4000 bytes
+    // the client writes 8000 bytes
+    const CLIENT_WRITES: usize = 8000;
+    // this gets split across 8 packets
+    // the first packet is stored in the Incoming
+    // the next three are incoming-buffered, bringing the incoming buffer size to 3600 bytes
+    // the last four are dropped due to the buffering limit and must be retransmitted
+    const EXPECTED_DROPPED: u64 = 4;
+
+    let _guard = subscribe();
+    let mut server_config = server_config();
+    configure_server(&mut server_config);
+    let mut pair = Pair::new(Arc::new(EndpointConfig::default()), server_config);
+    let config = client_config();
+
+    // Establish normal connection
+    let client_ch = pair.begin_connect(config.clone());
+    pair.drive();
+    pair.server.assert_accept();
+    pair.client
+        .connections
+        .get_mut(&client_ch)
+        .unwrap()
+        .close(pair.time, VarInt(0), [][..].into());
+    pair.drive();
+
+    pair.client.addr = SocketAddr::new(
+        Ipv6Addr::LOCALHOST.into(),
+        CLIENT_PORTS.lock().unwrap().next().unwrap(),
+    );
+    info!("resuming session");
+    pair.server.incoming_connection_behavior = IncomingConnectionBehavior::Wait;
+    let client_ch = pair.begin_connect(config);
+    assert!(pair.client_conn_mut(client_ch).has_0rtt());
+    let s = pair.client_streams(client_ch).open(Dir::Uni).unwrap();
+    pair.client_send(client_ch, s)
+        .write(&vec![0; CLIENT_WRITES])
+        .unwrap();
+    pair.drive();
+    let incoming = pair.server.waiting_incoming.pop().unwrap();
+    assert!(pair.server.waiting_incoming.is_empty());
+    let _ = pair.server.try_accept(incoming, pair.time);
+    pair.drive();
+
+    assert_matches!(
+        pair.client_conn_mut(client_ch).poll(),
+        Some(Event::HandshakeDataReady)
+    );
+    assert_matches!(
+        pair.client_conn_mut(client_ch).poll(),
+        Some(Event::Connected)
+    );
+
+    assert!(pair.client_conn_mut(client_ch).accepted_0rtt());
+    let server_ch = pair.server.assert_accept();
+
+    assert_matches!(
+        pair.server_conn_mut(server_ch).poll(),
+        Some(Event::HandshakeDataReady)
+    );
+    // We don't currently preserve stream event order wrt. connection events
+    assert_matches!(
+        pair.server_conn_mut(server_ch).poll(),
+        Some(Event::Connected)
+    );
+    assert_matches!(
+        pair.server_conn_mut(server_ch).poll(),
+        Some(Event::Stream(StreamEvent::Opened { dir: Dir::Uni }))
+    );
+
+    let mut recv = pair.server_recv(server_ch, s);
+    let mut chunks = recv.read(false).unwrap();
+    let mut offset = 0;
+    loop {
+        match chunks.next(usize::MAX) {
+            Ok(Some(chunk)) => {
+                assert_eq!(chunk.offset as usize, offset);
+                offset += chunk.bytes.len();
+            }
+            Err(ReadError::Blocked) => break,
+            Ok(None) => panic!("unexpected stream end"),
+            Err(e) => panic!("{}", e),
+        }
+    }
+    assert_eq!(offset, CLIENT_WRITES);
+    let _ = chunks.finalize();
+    assert_eq!(
+        pair.client_conn_mut(client_ch).lost_packets(),
+        EXPECTED_DROPPED
+    );
+}
+
+#[test]
+fn zero_rtt_incoming_buffer_size() {
+    test_zero_rtt_incoming_limit(|config| {
+        config.incoming_buffer_size(4000);
+    });
+}
+
+#[test]
+fn zero_rtt_incoming_buffer_size_total() {
+    test_zero_rtt_incoming_limit(|config| {
+        config.incoming_buffer_size_total(4000);
+    });
+}
+
 #[test]
 fn alpn_success() {
     let _guard = subscribe();
diff --git a/quinn-proto/src/tests/util.rs b/quinn-proto/src/tests/util.rs
index a7e4b92b2c..f10b9a2a9f 100644
--- a/quinn-proto/src/tests/util.rs
+++ b/quinn-proto/src/tests/util.rs
@@ -298,6 +298,7 @@ pub(super) struct TestEndpoint {
     pub(super) captured_packets: Vec<Vec<u8>>,
     pub(super) capture_inbound_packets: bool,
     pub(super) incoming_connection_behavior: IncomingConnectionBehavior,
+    pub(super) waiting_incoming: Vec<Incoming>,
 }
 
 #[derive(Debug, Copy, Clone)]
@@ -305,6 +306,7 @@ pub(super) enum IncomingConnectionBehavior {
     AcceptAll,
     RejectAll,
     Validate,
+    Wait,
 }
 
 impl TestEndpoint {
@@ -332,6 +334,7 @@ impl TestEndpoint {
             captured_packets: Vec::new(),
             capture_inbound_packets: false,
             incoming_connection_behavior: IncomingConnectionBehavior::AcceptAll,
+            waiting_incoming: Vec::new(),
         }
     }
 
@@ -374,6 +377,9 @@ impl TestEndpoint {
                                     self.retry(incoming);
                                 }
                             }
+                            IncomingConnectionBehavior::Wait => {
+                                self.waiting_incoming.push(incoming);
+                            }
                         }
                     }
                     DatagramEvent::ConnectionEvent(ch, event) => {