Repository: netty/netty
Issue: Netty Threads consume 100% of the CPU after update to the latest version
Labels: regression

Hello, after the update:

netty 4.1.54.Final -> 4.1.55.Final
boring ssl 2.0.34.Final -> 2.0.35.Final

During one of the test cases with file upload, we found netty epoll threads start consuming 100% of the CPU.
Stack trace:

```
"epollEventLoopGroup-3-1" #20 prio=10 os_prio=0 cpu=597400.03ms elapsed=15396.13s tid=0x00007f53c800a000 nid=0x664a0 runnable  [0x00007f53c33fc000]
   java.lang.Thread.State: RUNNABLE
        at io.netty.buffer.AbstractReferenceCountedByteBuf$1.unsafeOffset(AbstractReferenceCountedByteBuf.java:40)
        at io.netty.util.internal.ReferenceCountUpdater.isLiveNonVolatile(ReferenceCountUpdater.java:88)
        at io.netty.buffer.AbstractReferenceCountedByteBuf.isAccessible(AbstractReferenceCountedByteBuf.java:56)
        at io.netty.buffer.AbstractByteBuf.ensureAccessible(AbstractByteBuf.java:1489)
        at io.netty.buffer.AbstractByteBuf.checkIndex(AbstractByteBuf.java:1419)
        at io.netty.buffer.AbstractByteBuf.checkIndex(AbstractByteBuf.java:1415)
        at io.netty.buffer.AbstractByteBuf.getByte(AbstractByteBuf.java:357)
        at io.netty.buffer.CompositeByteBuf._getByte(CompositeByteBuf.java:950)
        at io.netty.buffer.UnpooledSlicedByteBuf._getByte(UnpooledSlicedByteBuf.java:39)
        at io.netty.buffer.AbstractByteBuf.firstIndexOf(AbstractByteBuf.java:1268)
        at io.netty.buffer.AbstractByteBuf.indexOf(AbstractByteBuf.java:1254)
        at io.netty.buffer.AbstractByteBuf.bytesBefore(AbstractByteBuf.java:1306)
        at io.netty.handler.codec.http.multipart.HttpPostMultipartRequestDecoder.findDelimiter(HttpPostMultipartRequestDecoder.java:1129)
        at io.netty.handler.codec.http.multipart.HttpPostMultipartRequestDecoder.loadDataMultipart(HttpPostMultipartRequestDecoder.java:1171)
        at io.netty.handler.codec.http.multipart.HttpPostMultipartRequestDecoder.decodeMultipart(HttpPostMultipartRequestDecoder.java:552)
        at io.netty.handler.codec.http.multipart.HttpPostMultipartRequestDecoder.findMultipartDisposition(HttpPostMultipartRequestDecoder.java:795)
        at io.netty.handler.codec.http.multipart.HttpPostMultipartRequestDecoder.decodeMultipart(HttpPostMultipartRequestDecoder.java:504)
        at io.netty.handler.codec.http.multipart.HttpPostMultipartRequestDecoder.findMultipartDelimiter(HttpPostMultipartRequestDecoder.java:656)
        at io.netty.handler.codec.http.multipart.HttpPostMultipartRequestDecoder.decodeMultipart(HttpPostMultipartRequestDecoder.java:491)
        at io.netty.handler.codec.http.multipart.HttpPostMultipartRequestDecoder.parseBodyMultipart(HttpPostMultipartRequestDecoder.java:456)
        at io.netty.handler.codec.http.multipart.HttpPostMultipartRequestDecoder.parseBody(HttpPostMultipartRequestDecoder.java:425)
        at io.netty.handler.codec.http.multipart.HttpPostMultipartRequestDecoder.offer(HttpPostMultipartRequestDecoder.java:353)

"epollEventLoopGroup-3-2" #21 prio=10 os_prio=0 cpu=434998.54ms elapsed=15396.10s tid=0x00007f53c800c800 nid=0x664a1 runnable  [0x00007f53c32fb000]
   java.lang.Thread.State: RUNNABLE
        at io.netty.buffer.AbstractReferenceCountedByteBuf$1.unsafeOffset(AbstractReferenceCountedByteBuf.java:40)
        at io.netty.util.internal.ReferenceCountUpdater.isLiveNonVolatile(ReferenceCountUpdater.java:88)
        at io.netty.buffer.AbstractReferenceCountedByteBuf.isAccessible(AbstractReferenceCountedByteBuf.java:56)
        at io.netty.buffer.AbstractByteBuf.ensureAccessible(AbstractByteBuf.java:1489)
        at io.netty.buffer.AbstractByteBuf.checkIndex(AbstractByteBuf.java:1419)
        at io.netty.buffer.AbstractByteBuf.checkIndex(AbstractByteBuf.java:1415)
        at io.netty.buffer.AbstractByteBuf.getByte(AbstractByteBuf.java:357)
        at io.netty.buffer.CompositeByteBuf._getByte(CompositeByteBuf.java:950)
        at io.netty.buffer.UnpooledSlicedByteBuf._getByte(UnpooledSlicedByteBuf.java:39)
        at io.netty.buffer.AbstractByteBuf.firstIndexOf(AbstractByteBuf.java:1268)
        at io.netty.buffer.AbstractByteBuf.indexOf(AbstractByteBuf.java:1254)
        at io.netty.buffer.AbstractByteBuf.bytesBefore(AbstractByteBuf.java:1306)
        at io.netty.handler.codec.http.multipart.HttpPostMultipartRequestDecoder.findDelimiter(HttpPostMultipartRequestDecoder.java:1129)
        at io.netty.handler.codec.http.multipart.HttpPostMultipartRequestDecoder.loadDataMultipart(HttpPostMultipartRequestDecoder.java:1171)
        at io.netty.handler.codec.http.multipart.HttpPostMultipartRequestDecoder.decodeMultipart(HttpPostMultipartRequestDecoder.java:552)
        at io.netty.handler.codec.http.multipart.HttpPostMultipartRequestDecoder.findMultipartDisposition(HttpPostMultipartRequestDecoder.java:795)
        at io.netty.handler.codec.http.multipart.HttpPostMultipartRequestDecoder.decodeMultipart(HttpPostMultipartRequestDecoder.java:504)
        at io.netty.handler.codec.http.multipart.HttpPostMultipartRequestDecoder.findMultipartDelimiter(HttpPostMultipartRequestDecoder.java:656)
        at io.netty.handler.codec.http.multipart.HttpPostMultipartRequestDecoder.decodeMultipart(HttpPostMultipartRequestDecoder.java:491)
        at io.netty.handler.codec.http.multipart.HttpPostMultipartRequestDecoder.parseBodyMultipart(HttpPostMultipartRequestDecoder.java:456)
        at io.netty.handler.codec.http.multipart.HttpPostMultipartRequestDecoder.parseBody(HttpPostMultipartRequestDecoder.java:425)
        at io.netty.handler.codec.http.multipart.HttpPostMultipartRequestDecoder.offer(HttpPostMultipartRequestDecoder.java:353)

"epollEventLoopGroup-2-1" #19 prio=10 os_prio=0 cpu=461.08ms elapsed=15396.27s tid=0x00007f53fcd00800 nid=0x6649f runnable  [0x00007f53e0902000]
   java.lang.Thread.State: RUNNABLE
        at io.netty.channel.epoll.Native.epollWait(Native Method)
        at io.netty.channel.epoll.Native.epollWait(Native.java:148)
        at io.netty.channel.epoll.Native.epollWait(Native.java:141)
        at io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:290)
        at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:347)
        at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)
        at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
        at java.lang.Thread.run(java.base@11.0.9.1/Thread.java:834)

"Common-Cleaner" #9 daemon prio=8 os_prio=0 cpu=16.86ms elapsed=15451.77s tid=0x00007f53fc120800 nid=0x66489 in Object.wait()  [0x00007f53e3323000]
   java.lang.Thread.State: TIMED_WAITING (on object monitor)
        at java.lang.Object.wait(java.base@11.0.9.1/Native Method)
        - waiting on <no object reference available>
        at java.lang.ref.ReferenceQueue.remove(java.base@11.0.9.1/ReferenceQueue.java:155)
        - waiting to re-lock in wait() <0x00000000eb5d8040> (a java.lang.ref.ReferenceQueue$Lock)
        at jdk.internal.ref.CleanerImpl.run(java.base@11.0.9.1/CleanerImpl.java:148)
        at java.lang.Thread.run(java.base@11.0.9.1/Thread.java:834)
        at jdk.internal.misc.InnocuousThread.run(java.base@11.0.9.1/InnocuousThread.java:134)

"Finalizer" #3 daemon prio=8 os_prio=0 cpu=4.08ms elapsed=15451.86s tid=0x00007f53fc0c2000 nid=0x66482 in Object.wait()  [0x00007f53e3d11000]
   java.lang.Thread.State: WAITING (on object monitor)
        at java.lang.Object.wait(java.base@11.0.9.1/Native Method)
        - waiting on <no object reference available>
        at java.lang.ref.ReferenceQueue.remove(java.base@11.0.9.1/ReferenceQueue.java:155)
        - waiting to re-lock in wait() <0x00000000eb5d7850> (a java.lang.ref.ReferenceQueue$Lock)
        at java.lang.ref.ReferenceQueue.remove(java.base@11.0.9.1/ReferenceQueue.java:176)
        at java.lang.ref.Finalizer$FinalizerThread.run(java.base@11.0.9.1/Finalizer.java:170)

"Reference Handler" #2 daemon prio=10 os_prio=0 cpu=5.52ms elapsed=15451.86s tid=0x00007f53fc0c0000 nid=0x66481 waiting on condition  [0x00007f53e3e12000]
   java.lang.Thread.State: RUNNABLE
        at java.lang.ref.Reference.waitForReferencePendingList(java.base@11.0.9.1/Native Method)
        at java.lang.ref.Reference.processPendingReferences(java.base@11.0.9.1/Reference.java:241)
        at java.lang.ref.Reference$ReferenceHandler.run(java.base@11.0.9.1/Reference.java:213)

```

I'll continue to investigate, however, maybe you have some clues.

openjdk version "11.0.9.1" 2020-11-04



Issue comments:
Can you share the code to reproduce?
Experiencing this as well

In this code:

```
    private static int findDelimiter(ByteBuf undecodedChunk, String delimiter, int offset) {
        ....
        while (delimiterNotFound && newOffset + delimeterLength <= toRead) {
```

`newOffset` bounces between 9 and 4

The unencoded chunk is:

```
bar-stream
--c2b0cb9ac1cf7cc7
content-disposition: form-data; name="data"; filename="data.json"
content-length: 16
content-type: application/json; charset=UTF-8

{"title":"Test"}
--c2b0cb9ac1cf7cc7--
```

Delimiter is `--c2b0cb9ac1cf7cc7`

Offset is `0`

I think the issue is `newOffset = posFirstChar + offset;` should be `newOffset = posFirstChar + newOffset;`
@jameskleeh You might be right ! Sorry for that...
Could you check the Junit test with this change?
Same here! 100% CPU on 4.1.55, but same code works fine on 4.1.54! 🤷🏻‍♂️

Pull request review comments:

Patch that resolved the issue:
diff --git a/codec-http/src/main/java/io/netty/handler/codec/http/multipart/HttpPostMultipartRequestDecoder.java b/codec-http/src/main/java/io/netty/handler/codec/http/multipart/HttpPostMultipartRequestDecoder.java
index b125dfae0e7..9e82ad7caf0 100644
--- a/codec-http/src/main/java/io/netty/handler/codec/http/multipart/HttpPostMultipartRequestDecoder.java
+++ b/codec-http/src/main/java/io/netty/handler/codec/http/multipart/HttpPostMultipartRequestDecoder.java
@@ -1132,7 +1132,7 @@ private static int findDelimiter(ByteBuf undecodedChunk, String delimiter, int o
                 newOffset = toRead;
                 return -newOffset;
             }
-            newOffset = posFirstChar + offset;
+            newOffset = posFirstChar + newOffset;
             if (newOffset + delimeterLength > toRead) {
                 return -newOffset;
             }


Test patch:
diff --git a/codec-http/src/test/java/io/netty/handler/codec/http/multipart/HttpPostRequestDecoderTest.java b/codec-http/src/test/java/io/netty/handler/codec/http/multipart/HttpPostRequestDecoderTest.java
index 45b8461ce72..acbcf1219d3 100644
--- a/codec-http/src/test/java/io/netty/handler/codec/http/multipart/HttpPostRequestDecoderTest.java
+++ b/codec-http/src/test/java/io/netty/handler/codec/http/multipart/HttpPostRequestDecoderTest.java
@@ -938,4 +938,42 @@ public void testDecodeFullHttpRequestWithUrlEncodedBodyWithInvalidHexNibbleLo()
             assertTrue(req.release());
         }
     }
+
+    @Test
+    public void testDecodeMultipartRequest() {
+        byte[] bodyBytes = ("--be38b42a9ad2713f\n" +
+                "content-disposition: form-data; name=\"title\"\n" +
+                "content-length: 10\n" +
+                "content-type: text/plain; charset=UTF-8\n" +
+                "\n" +
+                "bar-stream\n" +
+                "--be38b42a9ad2713f\n" +
+                "content-disposition: form-data; name=\"data\"; filename=\"data.json\"\n" +
+                "content-length: 16\n" +
+                "content-type: application/json; charset=UTF-8\n" +
+                "\n" +
+                "{\"title\":\"Test\"}\n" +
+                "--be38b42a9ad2713f--").getBytes();
+        ByteBuf content = Unpooled.directBuffer(bodyBytes.length);
+        content.writeBytes(bodyBytes);
+        FullHttpRequest req = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, "/", content);
+        req.headers().add("Content-Type", "multipart/form-data;boundary=be38b42a9ad2713f");
+
+        try {
+            HttpPostRequestDecoder decoder = new HttpPostRequestDecoder(new DefaultHttpDataFactory(false), req);
+            InterfaceHttpData data = decoder.getBodyHttpData("title");
+            assertTrue(data instanceof MemoryAttribute);
+            assertEquals("bar-stream", ((MemoryAttribute) data).getString());
+            assertTrue(data.release());
+            data = decoder.getBodyHttpData("data");
+            assertTrue(data instanceof MemoryFileUpload);
+            assertEquals("{\"title\":\"Test\"}", ((MemoryFileUpload) data).getString());
+            assertTrue(data.release());
+            decoder.destroy();
+        } catch (HttpPostRequestDecoder.ErrorDataDecoderException e) {
+            fail("Was not expecting an exception");
+        } finally {
+            assertTrue(req.release());
+        }
+    }
 }