Repository: bearcove/loona (language: rust)

Issue:
Allow building against non-uring tokio
This could be useful for folks who want to contribute from macOS/Windows.

The `WriteOwned` / `ReadOwned` traits are easier to implement against tokio than the other way around.

Here are the sticking points:

  * `read` / `write` in tokio land take `&mut` - even thought that's silly for sockets, tbqh. We have an unclear story around read/write ends of a socket right now in hring, I think we could clean it up - at least in h2, it's cleanly separated (at last).
  * We can no longer do `tokio_uring::spawn` in any of the core hring crates - we'll have to abstract over `tokio_uring::spawn` and, most probably [spawn_local](https://docs.rs/tokio/latest/tokio/task/fn.spawn_local.html) (that's what [tokio_uring's spawn](https://github.com/tokio-rs/tokio-uring/blob/575d864c2675863e2e69c6262846d6144f0ccc21/src/runtime/mod.rs#L61-L63) does right now anyway). - that means hring's non-uring version will only work inside of a LocalSet, a limitation I'm fine with (it's all build around Rc, not Arc)

I think that's most of it. It should be a relatively small investment and it could be interesting to compare hring with and without io_uring, compare the number of syscalls, throughput etc.


Discussion / hints:

Fix patch:
diff --git a/crates/hring-buffet/Cargo.toml b/crates/hring-buffet/Cargo.toml
index 5c1a429d..da3337b6 100644
--- a/crates/hring-buffet/Cargo.toml
+++ b/crates/hring-buffet/Cargo.toml
@@ -13,9 +13,8 @@ Buffer management for the `hring` crate.
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [features]
-default = ["non-uring"]
+default = ["tokio-uring"]
 miri = []
-non-uring = ["tokio/net", "tokio/rt"]
 
 [dependencies]
 eyre = "0.6.8"
@@ -26,7 +25,7 @@ nom = "7.1.3"
 pretty-hex = "0.3.0"
 thiserror = { version = "1.0.38", default-features = false }
 tokio = { version = "1.25.0", features = ["sync", "macros"] }
-tokio-uring = "0.4.0"
+tokio-uring = { version = "0.4.0", optional = true }
 tracing = "0.1.37"
 
 [dev-dependencies]
diff --git a/crates/hring/Cargo.toml b/crates/hring/Cargo.toml
index 8f974986..4da0ea19 100644
--- a/crates/hring/Cargo.toml
+++ b/crates/hring/Cargo.toml
@@ -10,6 +10,11 @@ description = """
 An HTTP implementation on top of io_uring
 """
 
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[features]
+default = ["tokio-uring"]
+
 [dependencies]
 byteorder = "1.4.3"
 enum-repr = "0.2.6"
@@ -25,7 +30,7 @@ pretty-hex = { version = "0.3.0", default-features = false }
 smallvec = { version = "1.10.0", default-features = false, features = ["const_generics", "const_new", "union"] }
 thiserror = { version = "1.0.38", default-features = false }
 tokio = { version = "1.24.2", features = ["macros", "sync"] }
-tokio-uring = "0.4.0"
+tokio-uring = { version = "0.4.0", optional = true }
 tracing = { version = "0.1.37", default-features = false }
 
 [dev-dependencies]
@@ -38,6 +43,3 @@ tokio = { version = "1.24.2", default-features = false, features = ["io-util", "
 futures-util = { version = "0.3.25", default-features = false, features = ["std"] }
 curl = { version = "0.4.44", default-features = false, features = ["http2"] }
 libc = "0.2.139"
-
-[features]
-non-uring = []
diff --git a/crates/hring/src/h2/encode.rs b/crates/hring/src/h2/encode.rs
index 21f2d9e5..7b3c6429 100644
--- a/crates/hring/src/h2/encode.rs
+++ b/crates/hring/src/h2/encode.rs
@@ -138,7 +138,7 @@ impl Drop for H2Encoder {
 
         if !evs.is_empty() {
             let tx = self.tx.clone();
-            tokio_uring::spawn(async move {
+            crate::spawn(async move {
                 for ev in evs {
                     if tx.send(ev).await.is_err() {
                         warn!("could not send event to h2 connection handler");
diff --git a/crates/hring/src/h2/server.rs b/crates/hring/src/h2/server.rs
index 1e7bc6d3..bda12567 100644
--- a/crates/hring/src/h2/server.rs
+++ b/crates/hring/src/h2/server.rs
@@ -1021,7 +1021,7 @@ fn end_headers(
                 rx: piece_rx,
             };
 
-            tokio_uring::spawn({
+            crate::spawn({
                 let driver = driver.clone();
                 async move {
                     let mut req_body = req_body;
diff --git a/crates/hring/src/lib.rs b/crates/hring/src/lib.rs
index 741deaf9..b262b4e1 100644
--- a/crates/hring/src/lib.rs
+++ b/crates/hring/src/lib.rs
@@ -11,6 +11,8 @@ pub use types::*;
 pub mod h1;
 pub mod h2;
 
+use std::future::Future;
+
 mod responder;
 pub use responder::*;
 
@@ -20,8 +22,22 @@ pub use hring_buffet as buffet;
 pub use http;
 
 /// re-exported so consumers can use whatever forked version we use
+#[cfg(feature = "tokio-uring")]
 pub use tokio_uring;
 
+/// Spawns a new asynchronous task, returning a [`JoinHandle`] for it.
+///
+/// Spawning a task enables the task to execute concurrently to other tasks.
+/// There is no guarantee that a spawned task will execute to completion. When a
+/// runtime is shutdown, all outstanding tasks are dropped, regardless of the
+/// lifecycle of that task.
+///
+/// This function must be called from the context of a `tokio-uring` runtime,
+/// or a tokio local set (at the time of this writing, they're the same thing).
+pub fn spawn<T: Future + 'static>(task: T) -> tokio::task::JoinHandle<T::Output> {
+    tokio::task::spawn_local(task)
+}
+
 pub trait ServerDriver {
     async fn handle<E: Encoder>(
         &self,


Test patch:
diff --git a/test-crates/hring-h2spec/Cargo.toml b/test-crates/hring-h2spec/Cargo.toml
index 939fe087..2e3627ee 100644
--- a/test-crates/hring-h2spec/Cargo.toml
+++ b/test-crates/hring-h2spec/Cargo.toml
@@ -6,8 +6,8 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+hring = { version = "0.1.0", path = "../../crates/hring", default-features = false }
 color-eyre = "0.6.2"
-hring = { version = "0.1.0", path = "../../crates/hring" }
 tokio = { version = "1.23.0", features = ["full"] }
 tracing = "0.1.37"
 tracing-subscriber = { version = "0.3.16", features = ["env-filter"] }
@@ -17,5 +17,5 @@ which = "4.4.0"
 opt-level = 2
 
 [features]
-# default = ["non-uring"]
-non-uring = ["hring/non-uring"]
+default = ["tokio-uring"]
+tokio-uring = ["hring/tokio-uring"]
diff --git a/test-crates/hring-h2spec/src/main.rs b/test-crates/hring-h2spec/src/main.rs
index a4c312eb..c56adce1 100644
--- a/test-crates/hring-h2spec/src/main.rs
+++ b/test-crates/hring-h2spec/src/main.rs
@@ -12,15 +12,16 @@ use tokio::process::Command;
 use tracing::{error, info};
 use tracing_subscriber::EnvFilter;
 
-#[cfg(not(feature = "non-uring"))]
+#[cfg(feature = "tokio-uring")]
 mod uring;
-#[cfg(not(feature = "non-uring"))]
+
+#[cfg(feature = "tokio-uring")]
 use uring as server_impl;
 
-#[cfg(feature = "non-uring")]
+#[cfg(not(feature = "tokio-uring"))]
 mod non_uring;
 
-#[cfg(feature = "non-uring")]
+#[cfg(not(feature = "tokio-uring"))]
 use non_uring as server_impl;
 
 fn main() {
@@ -43,7 +44,24 @@ fn main() {
         }
     };
 
-    hring::tokio_uring::start(async move { real_main(h2spec_binary).await.unwrap() })
+    #[cfg(feature = "tokio-uring")]
+    hring::tokio_uring::start(async move { real_main(h2spec_binary).await.unwrap() });
+
+    #[cfg(not(feature = "tokio-uring"))]
+    {
+        use tokio::task::LocalSet;
+
+        tokio::runtime::Builder::new_multi_thread()
+            .enable_all()
+            .build()
+            .unwrap()
+            .block_on(async move {
+                let local = LocalSet::new();
+                local
+                    .run_until(async move { real_main(h2spec_binary).await.unwrap() })
+                    .await;
+            })
+    }
 }
 
 struct SDriver;
diff --git a/test-crates/hring-h2spec/src/non_uring.rs b/test-crates/hring-h2spec/src/non_uring.rs
index da93d007..03fa61b5 100644
--- a/test-crates/hring-h2spec/src/non_uring.rs
+++ b/test-crates/hring-h2spec/src/non_uring.rs
@@ -1,6 +1,6 @@
 use std::{net::SocketAddr, rc::Rc};
 
-use hring::buffet::{RollMut, SplitOwned};
+use hring::buffet::RollMut;
 use tokio::net::TcpListener;
 
 use crate::SDriver;
@@ -24,7 +24,7 @@ pub(crate) async fn run_server(ln: TcpListener) -> color_eyre::Result<()> {
         let driver = Rc::new(SDriver);
 
         tokio::task::spawn_local(async move {
-            if let Err(e) = hring::h2::serve(stream.split_owned(), conf, client_buf, driver).await {
+            if let Err(e) = hring::h2::serve(stream.into_split(), conf, client_buf, driver).await {
                 tracing::error!("error serving client {}: {}", addr, e);
             }
         });