Repository: netty/netty
Pull request #14908: IoUring: Allow to enable / disable multishot via a system property
Author: login: normanmaurer
Created: 2025-03-10T01:32:55Z | Merged: 2025-03-10T22:05:45Z

Motivation:

We should support to disable multishot variants via a system property so people can workaround bugs and we can test all code-paths

Modifications:

Allow to enable / disable via system property

Result:

More flexible configuration


Diff:
diff --git a/transport-classes-io_uring/src/main/java/io/netty/channel/uring/AbstractIoUringChannel.java b/transport-classes-io_uring/src/main/java/io/netty/channel/uring/AbstractIoUringChannel.java
index 8d8aec65250..f8ba483a817 100644
--- a/transport-classes-io_uring/src/main/java/io/netty/channel/uring/AbstractIoUringChannel.java
+++ b/transport-classes-io_uring/src/main/java/io/netty/channel/uring/AbstractIoUringChannel.java
@@ -728,7 +728,7 @@ final void schedulePollIn() {
             if (!isActive() || shouldBreakIoUringInReady(config())) {
                 return;
             }
-            pollInId = schedulePollAdd(POLL_IN_SCHEDULED, Native.POLLIN, IoUring.isPollAddMultishotSupported());
+            pollInId = schedulePollAdd(POLL_IN_SCHEDULED, Native.POLLIN, IoUring.isPollAddMultishotEnabled());
         }
 
         private void readComplete(byte op, int res, int flags, short data) {
diff --git a/transport-classes-io_uring/src/main/java/io/netty/channel/uring/AbstractIoUringServerChannel.java b/transport-classes-io_uring/src/main/java/io/netty/channel/uring/AbstractIoUringServerChannel.java
index 40c3dbc1fdb..b72f4c557c5 100644
--- a/transport-classes-io_uring/src/main/java/io/netty/channel/uring/AbstractIoUringServerChannel.java
+++ b/transport-classes-io_uring/src/main/java/io/netty/channel/uring/AbstractIoUringServerChannel.java
@@ -77,7 +77,7 @@ protected AbstractIoUringServerChannel(LinuxSocket socket, boolean active) {
         //       time to process a past connection. If the application knows that a
         //       new connection cannot come in before a previous one has been
         //       processed, it may be used as expected.
-        if (IoUring.isAcceptMultishotSupported()) {
+        if (IoUring.isAcceptMultishotEnabled()) {
             acceptedAddressMemory = null;
         } else {
             acceptedAddressMemory = new AcceptedAddressMemory();
@@ -154,7 +154,7 @@ protected int scheduleRead0(boolean first, boolean socketIsEmpty) {
 
             final long acceptedAddressMemoryAddress;
             final long acceptedAddressLengthMemoryAddress;
-            if (IoUring.isAcceptMultishotSupported()) {
+            if (IoUring.isAcceptMultishotEnabled()) {
                 // Let's use multi-shot accept to reduce overhead.
                 ioPrio = Native.IORING_ACCEPT_MULTISHOT;
                 acceptedAddressMemoryAddress = 0;
diff --git a/transport-classes-io_uring/src/main/java/io/netty/channel/uring/AbstractIoUringStreamChannel.java b/transport-classes-io_uring/src/main/java/io/netty/channel/uring/AbstractIoUringStreamChannel.java
index 5b98c6751e8..970d1e2cc11 100644
--- a/transport-classes-io_uring/src/main/java/io/netty/channel/uring/AbstractIoUringStreamChannel.java
+++ b/transport-classes-io_uring/src/main/java/io/netty/channel/uring/AbstractIoUringStreamChannel.java
@@ -355,7 +355,7 @@ protected int scheduleRead0(boolean first, boolean socketIsEmpty) {
         private int scheduleReadProviderBuffer(IoUringBufferRing bufferRing, boolean first, boolean socketIsEmpty) {
             short bgId = bufferRing.bufferGroupId();
             try {
-                boolean multishot = IoUring.isRecvMultishotSupported();
+                boolean multishot = IoUring.isRecvMultishotEnabled();
                 byte flags = (byte) Native.IOSQE_BUFFER_SELECT;
                 short ioPrio;
                 final int recvFlags;
@@ -368,7 +368,7 @@ private int scheduleReadProviderBuffer(IoUringBufferRing bufferRing, boolean fir
                     ioPrio = calculateRecvIoPrio(first, socketIsEmpty);
                     recvFlags = calculateRecvFlags(first);
                 }
-                if (IoUring.isRecvsendBundleSupported()) {
+                if (IoUring.isRecvsendBundleEnabled()) {
                     // See https://github.com/axboe/liburing/wiki/
                     // What's-new-with-io_uring-in-6.10#add-support-for-sendrecv-bundles
                     ioPrio |= Native.IORING_RECVSEND_BUNDLE;
@@ -446,7 +446,7 @@ protected void readComplete0(byte op, int res, int flags, short data, int outsta
                 } else if (res > 0) {
                     if (useBufferRing) {
 
-                        if (IoUring.isRecvsendBundleSupported()) {
+                        if (IoUring.isRecvsendBundleEnabled()) {
                             // If RECVSEND_BUNDLE is supported we need to do a bit more work here.
                             // In this case we might need to obtain multiple buffers out of the buffer ring as
                             // multiple of them might have been filled for one recv operation.
diff --git a/transport-classes-io_uring/src/main/java/io/netty/channel/uring/IoUring.java b/transport-classes-io_uring/src/main/java/io/netty/channel/uring/IoUring.java
index bb880bd1b77..66e7973455f 100644
--- a/transport-classes-io_uring/src/main/java/io/netty/channel/uring/IoUring.java
+++ b/transport-classes-io_uring/src/main/java/io/netty/channel/uring/IoUring.java
@@ -38,6 +38,10 @@ public final class IoUring {
     private static final boolean IORING_SETUP_DEFER_TASKRUN_SUPPORTED;
     private static final boolean IORING_REGISTER_BUFFER_RING_SUPPORTED;
     private static final boolean IORING_REGISTER_BUFFER_RING_INC_SUPPORTED;
+    private static final boolean IORING_ACCEPT_MULTISHOT_ENABLED;
+    private static final boolean IORING_RECV_MULTISHOT_ENABLED;
+    private static final boolean IORING_RECVSEND_BUNDLE_ENABLED;
+    private static final boolean IORING_POLL_ADD_MULTISHOT_ENABLED;
 
     private static final InternalLogger logger;
 
@@ -150,6 +154,15 @@ public final class IoUring {
         IORING_SETUP_DEFER_TASKRUN_SUPPORTED = deferTaskrunSupported;
         IORING_REGISTER_BUFFER_RING_SUPPORTED = registerBufferRingSupported;
         IORING_REGISTER_BUFFER_RING_INC_SUPPORTED = registerBufferRingIncSupported;
+
+        IORING_ACCEPT_MULTISHOT_ENABLED = IORING_ACCEPT_MULTISHOT_SUPPORTED && SystemPropertyUtil.getBoolean(
+                "io.netty.iouring.acceptMultiShotEnabled", true);
+        IORING_RECV_MULTISHOT_ENABLED = IORING_RECV_MULTISHOT_SUPPORTED && SystemPropertyUtil.getBoolean(
+                "io.netty.iouring.recvMultiShotEnabled", true);
+        IORING_RECVSEND_BUNDLE_ENABLED = IORING_RECVSEND_BUNDLE_SUPPORTED && SystemPropertyUtil.getBoolean(
+                "io.netty.iouring.recvsendBundleEnabled", true);
+       IORING_POLL_ADD_MULTISHOT_ENABLED = IORING_POLL_ADD_MULTISHOT_SUPPORTED && SystemPropertyUtil.getBoolean(
+               "io.netty.iouring.pollAddMultishotEnabled", true);
     }
 
     public static boolean isAvailable() {
@@ -180,7 +193,12 @@ static boolean isCqeFSockNonEmptySupported() {
         return IORING_CQE_F_SOCK_NONEMPTY_SUPPORTED;
     }
 
-    static boolean isSpliceSupported() {
+    /**
+     * Returns if SPLICE is supported or not.
+     *
+     * @return {@code true} if supported, {@code false} otherwise.
+     */
+    public static boolean isSpliceSupported() {
         return IORING_SPLICE_SUPPORTED;
     }
 
@@ -224,14 +242,60 @@ static boolean isSetupDeferTaskrunSupported() {
         return IORING_SETUP_DEFER_TASKRUN_SUPPORTED;
     }
 
+    /**
+     * Returns if it is supported to use a buffer ring.
+     *
+     * @return {@code true} if supported, {@code false} otherwise.
+     */
     public static boolean isRegisterBufferRingSupported() {
         return IORING_REGISTER_BUFFER_RING_SUPPORTED;
     }
 
+    /**
+     * Returns if it is supported to use an incremental buffer ring.
+     *
+     * @return {@code true} if supported, {@code false} otherwise.
+     */
     public static boolean isRegisterBufferRingIncSupported() {
         return IORING_REGISTER_BUFFER_RING_INC_SUPPORTED;
     }
 
+    /**
+     * Returns if multi-shot ACCEPT is used or not.
+     *
+     * @return {@code true} if enabled, {@code false} otherwise.
+     */
+    public static boolean isAcceptMultishotEnabled() {
+        return IORING_ACCEPT_MULTISHOT_ENABLED;
+    }
+
+    /**
+     * Returns if multi-shot RECV is used or not.
+     *
+     * @return {@code true} if enabled, {@code false} otherwise.
+     */
+    public static boolean isRecvMultishotEnabled() {
+        return IORING_RECV_MULTISHOT_ENABLED;
+    }
+
+    /**
+     * Returns if RECVSEND bundles are used or not.
+     *
+     * @return {@code true} if enabled, {@code false} otherwise.
+     */
+    public static boolean isRecvsendBundleEnabled() {
+        return IORING_RECVSEND_BUNDLE_ENABLED;
+    }
+
+    /**
+     * Returns if multi-shot POLL_ADD is used or not.
+     *
+     * @return {@code true} if enabled, {@code false} otherwise.
+     */
+    public static boolean isPollAddMultishotEnabled() {
+        return IORING_POLL_ADD_MULTISHOT_ENABLED;
+    }
+
     public static void ensureAvailability() {
         if (UNAVAILABILITY_CAUSE != null) {
             throw (Error) new UnsatisfiedLinkError(
diff --git a/transport-native-io_uring/src/test/java/io/netty/channel/uring/IoUringBufferRingTest.java b/transport-native-io_uring/src/test/java/io/netty/channel/uring/IoUringBufferRingTest.java
index 4d49e24d713..6d26d54e6a6 100644
--- a/transport-native-io_uring/src/test/java/io/netty/channel/uring/IoUringBufferRingTest.java
+++ b/transport-native-io_uring/src/test/java/io/netty/channel/uring/IoUringBufferRingTest.java
@@ -164,16 +164,16 @@ public void userEventTriggered(ChannelHandlerContext ctx, Object evt) {
         group.shutdownGracefully();
     }
 
-    static boolean recvsendBundleSupported() {
-        return IoUring.isRecvsendBundleSupported();
+    static boolean recvsendBundleEnabled() {
+        return IoUring.isRecvsendBundleEnabled();
     }
 
     @ParameterizedTest
     @ValueSource(booleans = {true, false})
-    @EnabledIf("recvsendBundleSupported")
+    @EnabledIf("recvsendBundleEnabled")
     public void testProviderBufferReadWithRecvsendBundle(boolean incremental) throws InterruptedException {
         // See https://lore.kernel.org/io-uring/184f9f92-a682-4205-a15d-89e18f664502@kernel.dk/T/#u
-        assumeTrue(IoUring.isRecvMultishotSupported(),
+        assumeTrue(IoUring.isRecvMultishotEnabled(),
                 "Only yields expected test results when using multishot atm");
         if (incremental) {
             assumeTrue(IoUring.isRegisterBufferRingIncSupported());


Reviews:
user: login: dreamlike-ocean
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-10T02:02:57Z
user: login: normanmaurer
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-10T15:11:44Z
user: login: franz1981
body: +1 from me 
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-10T15:56:06Z
user: login: chrisvest
state: APPROVED
author_association: MEMBER
submitted_at: 2025-03-10T17:07:52Z

Review comments:
diff_hunk: @@ -232,6 +245,22 @@ public static boolean isRegisterBufferRingIncSupported() {
         return IORING_REGISTER_BUFFER_RING_INC_SUPPORTED;
     }
 
+    static boolean isAcceptMultishotEnabled() {
path: transport-classes-io_uring/src/main/java/io/netty/channel/uring/IoUring.java
user: login: dreamlike-ocean
body: Perhaps making there methods to public could assist user in better adapting to different runtime environments
such as 
```java
if (!IoUinrg.isSpliceSupported) {
  pipeline.addlast(ChunkedWriteHandler);
}

if(IoUring.isRecvMultishotEnabled) {
  registerBufferRing(...)
}
```
created_at: 2025-03-10T02:02:57Z
author_association: CONTRIBUTOR
diff_hunk: @@ -232,6 +245,22 @@ public static boolean isRegisterBufferRingIncSupported() {
         return IORING_REGISTER_BUFFER_RING_INC_SUPPORTED;
     }
 
+    static boolean isAcceptMultishotEnabled() {
path: transport-classes-io_uring/src/main/java/io/netty/channel/uring/IoUring.java
user: login: normanmaurer
body: This makes sense... Let's make them public. 
created_at: 2025-03-10T15:11:44Z
author_association: MEMBER

PR comments:
user: login: normanmaurer
created_at: 2025-03-10T15:38:59Z
body: Alternative we could also provide `IoUringChannelOption` for these but this feels more like something that people will not mess with so I eel more like a system property is fine. @dreamlike-ocean @chrisvest WDYT ?
author_association: MEMBER
user: login: chrisvest
created_at: 2025-03-10T16:30:29Z
body: I think making them properties is fine in this case. By default, Netty should drive the transports as efficiently as possible. We can make channel options later, if need be.
author_association: MEMBER
user: login: franz1981
created_at: 2025-03-10T16:32:41Z
body: > By default, Netty should drive the transports as efficiently as possible. We can make channel options later, if need be.

exactly, the only reason why i see the sys pro used is for simplify testing (no more N kernels versions) and CVE mitigation/bugs (since it's easier than downgreating to an old kernel - but TBH it's rare and I believe can be done on the kernel too)
author_association: CONTRIBUTOR
user: login: dreamlike-ocean
created_at: 2025-03-10T16:35:37Z
body: I prefer system property,too
author_association: CONTRIBUTOR
user: login: chrisvest
created_at: 2025-03-10T17:07:36Z
body: From the linux-x86_64-java11 build:

```
2025-03-10T16:34:02.5046197Z 16:34:02.418 [testsuite-io_uring-worker-11-2] DEBUG io.netty.channel.uring.IoUringIoHandler -- ignoring POLL_ADD completion for unknown registration (id=-2147483631, res=-125)
2025-03-10T16:34:02.5052465Z 16:34:02.452 [main] INFO io.netty.channel.uring.IoUringSocketGatheringWriteTest -- Running: testGatheringWriteBig 3 of 3 (ServerBootstrap(ServerBootstrapConfig(group: MultiThreadIoEventLoopGroup, channelFactory: ReflectiveChannelFactory(NioServerSocketChannel.class), localAddress: /0:0:0:0:0:0:0:1%lo:0, options: {ALLOCATOR=AdaptiveByteBufAllocator(directByDefault: true)}, childGroup: MultiThreadIoEventLoopGroup, childOptions: {ALLOCATOR=AdaptiveByteBufAllocator(directByDefault: true)})) + Bootstrap(BootstrapConfig(group: MultiThreadIoEventLoopGroup, channelFactory: ReflectiveChannelFactory(IoUringSocketChannel.class), options: {ALLOCATOR=AdaptiveByteBufAllocator(directByDefault: true)}, resolver: io.netty.resolver.DefaultAddressResolverGroup@79c4715d))) with AdaptiveByteBufAllocator
2025-03-10T16:36:11.0036347Z /bin/sh: line 1: 50373 Aborted                 (core dumped) '/root/.sdkman/candidates/java/11.0.25-zulu/bin/java' '-D_' '-jar' '/code/transport-native-io_uring/target/surefire/surefirebooter-20250310163345399_5428.jar' '/code/transport-native-io_uring/target/surefire' '2025-03-10T15-28-36_858-jvmRun1' 'surefire-20250310163345399_5426tmp' 'surefire_36-20250310163345399_5427tmp'
```
author_association: MEMBER
user: login: normanmaurer
created_at: 2025-03-10T17:13:32Z
body: > From the linux-x86_64-java11 build:
> 
> ```
> 2025-03-10T16:34:02.5046197Z 16:34:02.418 [testsuite-io_uring-worker-11-2] DEBUG io.netty.channel.uring.IoUringIoHandler -- ignoring POLL_ADD completion for unknown registration (id=-2147483631, res=-125)
> 2025-03-10T16:34:02.5052465Z 16:34:02.452 [main] INFO io.netty.channel.uring.IoUringSocketGatheringWriteTest -- Running: testGatheringWriteBig 3 of 3 (ServerBootstrap(ServerBootstrapConfig(group: MultiThreadIoEventLoopGroup, channelFactory: ReflectiveChannelFactory(NioServerSocketChannel.class), localAddress: /0:0:0:0:0:0:0:1%lo:0, options: {ALLOCATOR=AdaptiveByteBufAllocator(directByDefault: true)}, childGroup: MultiThreadIoEventLoopGroup, childOptions: {ALLOCATOR=AdaptiveByteBufAllocator(directByDefault: true)})) + Bootstrap(BootstrapConfig(group: MultiThreadIoEventLoopGroup, channelFactory: ReflectiveChannelFactory(IoUringSocketChannel.class), options: {ALLOCATOR=AdaptiveByteBufAllocator(directByDefault: true)}, resolver: io.netty.resolver.DefaultAddressResolverGroup@79c4715d))) with AdaptiveByteBufAllocator
> 2025-03-10T16:36:11.0036347Z /bin/sh: line 1: 50373 Aborted                 (core dumped) '/root/.sdkman/candidates/java/11.0.25-zulu/bin/java' '-D_' '-jar' '/code/transport-native-io_uring/target/surefire/surefirebooter-20250310163345399_5428.jar' '/code/transport-native-io_uring/target/surefire' '2025-03-10T15-28-36_858-jvmRun1' 'surefire-20250310163345399_5426tmp' 'surefire_36-20250310163345399_5427tmp'
> ```

Yep... still looking it it...
author_association: MEMBER