Repository: netty/netty
Issue: Epoll HttpServer ClosedChannelException death spiral
Labels: defect

There appears to be something like a ClosedChannelException death spiral in conjunction with chunked HTTP POSTs and Epoll.

It starts if there is a ChannelHandler that throws an Exception and an another ChannelHandler that catches it and attempts to close the Channel. The server will then spin for a bit throwing ClosedChannelExceptions.

The same test using NIO doesn't exhibit this behavior.

```
Netty: 4.1.0.Beta6-SNAPSHOT

$ java -version
java version "1.8.0_40"
Java(TM) SE Runtime Environment (build 1.8.0_40-b25)
Java HotSpot(TM) 64-Bit Server VM (build 25.40-b25, mixed mode)

$ uname -a
Linux ardverk 4.0.7-2-ARCH #1 SMP PREEMPT Tue Jun 30 07:50:21 UTC 2015 x86_64 GNU/Linux
```

```
public class ChunkedPostWithEpollTest {

  private static final boolean USE_EPOLL = true;

  @Test
  public void httpChunkedPostWithEpollServer() throws Exception {
    Channel server = newServer(8080);

    try (CloseableHttpClient client = HttpClientBuilder.create().build()) {

      HttpPost post = new HttpPost("http://localhost:8080");
      post.setEntity(createChunkedOfSize(1));
      client.execute(post);

    } finally {
      server.close();
    }
  }

  private static Channel newServer(int port) throws Exception {
    ServerBootstrap serverBootstrap = new ServerBootstrap();

    EventLoopGroup acceptorGroup = USE_EPOLL 
        ? new EpollEventLoopGroup() 
        : new NioEventLoopGroup();

    EventLoopGroup workerGroup = USE_EPOLL 
        ? new EpollEventLoopGroup()
        : new NioEventLoopGroup();

    return serverBootstrap.group(acceptorGroup, workerGroup)
        .channel(USE_EPOLL 
            ? EpollServerSocketChannel.class 
            : NioServerSocketChannel.class)
        .childHandler(new MyInitializer())
        .bind(8080)
        .sync()
        .channel();
  }

  private static HttpEntity createChunkedOfSize(int mb) {
    int bytes = mb * 1024 * 1024;
    byte[] buffer = new byte[bytes];
    ByteArrayEntity entity = new ByteArrayEntity(buffer, ContentType.APPLICATION_OCTET_STREAM);
    entity.setChunked(true);
    return entity;
  }

  @Sharable
  private static class MyInitializer extends ChannelInitializer<Channel> {

    @Override
    protected void initChannel(Channel ch) throws Exception {
      ChannelPipeline pipeline = ch.pipeline();

      pipeline.addLast(new HttpServerCodec());
      pipeline.addLast(new BuggyChannelHandler());
      pipeline.addLast(new ExceptionHandler());
    }
  }

  @Sharable
  private static class BuggyChannelHandler extends ChannelInboundHandlerAdapter {

    @Override
    public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
      throw new NullPointerException("I am a bug!");
    }
  }

  @Sharable
  private static class ExceptionHandler extends ChannelInboundHandlerAdapter {

    @Override
    public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {

      cause.printStackTrace();

      ctx.close();
    }
  }
}
```

```
[main] DEBUG io.netty.channel.MultithreadEventLoopGroup - -Dio.netty.eventLoopThreads: 24
[main] DEBUG io.netty.util.NetUtil - Loopback interface: lo (lo, 0:0:0:0:0:0:0:1%lo)
[main] DEBUG io.netty.util.NetUtil - /proc/sys/net/core/somaxconn: 128
[main] DEBUG io.netty.channel.DefaultChannelId - -Dio.netty.processId: 9147 (auto-detected)
[main] DEBUG io.netty.channel.DefaultChannelId - -Dio.netty.machineId: 40:6c:8f:ff:fe:b9:25:55 (auto-detected)
[main] DEBUG io.netty.buffer.PooledByteBufAllocator - -Dio.netty.allocator.numHeapArenas: 24
[main] DEBUG io.netty.buffer.PooledByteBufAllocator - -Dio.netty.allocator.numDirectArenas: 24
[main] DEBUG io.netty.buffer.PooledByteBufAllocator - -Dio.netty.allocator.pageSize: 8192
[main] DEBUG io.netty.buffer.PooledByteBufAllocator - -Dio.netty.allocator.maxOrder: 11
[main] DEBUG io.netty.buffer.PooledByteBufAllocator - -Dio.netty.allocator.chunkSize: 16777216
[main] DEBUG io.netty.buffer.PooledByteBufAllocator - -Dio.netty.allocator.tinyCacheSize: 512
[main] DEBUG io.netty.buffer.PooledByteBufAllocator - -Dio.netty.allocator.smallCacheSize: 256
[main] DEBUG io.netty.buffer.PooledByteBufAllocator - -Dio.netty.allocator.normalCacheSize: 64
[main] DEBUG io.netty.buffer.PooledByteBufAllocator - -Dio.netty.allocator.maxCachedBufferCapacity: 32768
[main] DEBUG io.netty.buffer.PooledByteBufAllocator - -Dio.netty.allocator.cacheTrimInterval: 8192
[main] DEBUG io.netty.buffer.ByteBufUtil - -Dio.netty.allocator.type: pooled
[main] DEBUG io.netty.buffer.ByteBufUtil - -Dio.netty.threadLocalDirectBufferSize: 65536
[epollEventLoopGroup-3-1] DEBUG io.netty.util.ResourceLeakDetector - -Dio.netty.leakDetectionLevel: simple
[epollEventLoopGroup-3-1] DEBUG io.netty.util.Recycler - -Dio.netty.recycler.maxCapacity: 262144
java.lang.NullPointerException: I am a bug!
    at ChunkedPostWithEpollTest$BuggyChannelHandler.channelRead(ChunkedPostWithEpollTest.java:80)
    at io.netty.channel.ChannelHandlerInvokerUtil.invokeChannelReadNow(ChannelHandlerInvokerUtil.java:83)
    at io.netty.channel.DefaultChannelHandlerInvoker.invokeChannelRead(DefaultChannelHandlerInvoker.java:153)
    at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:157)
    at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:244)
    at io.netty.channel.ChannelHandlerInvokerUtil.invokeChannelReadNow(ChannelHandlerInvokerUtil.java:83)
    at io.netty.channel.DefaultChannelHandlerInvoker.invokeChannelRead(DefaultChannelHandlerInvoker.java:153)
    at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:157)
    at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:946)
    at io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:800)
    at io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:345)
    at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:253)
    at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:699)
    at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:137)
    at java.lang.Thread.run(Thread.java:745)
java.lang.NullPointerException: I am a bug!
    at ChunkedPostWithEpollTest$BuggyChannelHandler.channelRead(ChunkedPostWithEpollTest.java:80)
    at io.netty.channel.ChannelHandlerInvokerUtil.invokeChannelReadNow(ChannelHandlerInvokerUtil.java:83)
    at io.netty.channel.DefaultChannelHandlerInvoker.invokeChannelRead(DefaultChannelHandlerInvoker.java:153)
    at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:157)
    at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:244)
    at io.netty.channel.ChannelHandlerInvokerUtil.invokeChannelReadNow(ChannelHandlerInvokerUtil.java:83)
    at io.netty.channel.DefaultChannelHandlerInvoker.invokeChannelRead(DefaultChannelHandlerInvoker.java:153)
    at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:157)
    at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:946)
    at io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:800)
    at io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:345)
    at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:253)
    at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:699)
    at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:137)
    at java.lang.Thread.run(Thread.java:745)
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
```


Issue comments:
@rkapsi thx will check

@normanmaurer - I got this one.

Interesting, if I run that test from the CLI it's eventually starting to spew these ones...

```
java.nio.channels.ClosedChannelException
    java.nio.channels.ClosedChannelException
    java.nio.channels.ClosedChannelException
    java.io.IOException: readAddress() failed: Invalid argument
        at io.netty.channel.epoll.Native.newIOException(Native.java:129)
        at io.netty.channel.epoll.Native.ioResult(Native.java:145)
        at io.netty.channel.epoll.Native.readAddress(Native.java:286)
        at io.netty.channel.epoll.AbstractEpollChannel.doReadBytes(AbstractEpollChannel.java:244)
        at io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:790)
        at io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:345)
        at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:253)
        at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:699)
        at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:137)
        at java.lang.Thread.run(Thread.java:745)
    java.io.IOException: readAddress() failed: Invalid argument
        at io.netty.channel.epoll.Native.newIOException(Native.java:129)
        at io.netty.channel.epoll.Native.ioResult(Native.java:145)
        at io.netty.channel.epoll.Native.readAddress(Native.java:286)
        at io.netty.channel.epoll.AbstractEpollChannel.doReadBytes(AbstractEpollChannel.java:244)
        at io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:790)
        at io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:345)
        at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:253)
        at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:699)
        at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:137)
        at java.lang.Thread.run(Thread.java:745)
    java.io.IOException: readAddress() failed: Invalid argument
        at io.netty.channel.epoll.Native.newIOException(Native.java:129)
        at io.netty.channel.epoll.Native.ioResult(Native.java:145)
        at io.netty.channel.epoll.Native.readAddress(Native.java:286)
        at io.netty.channel.epoll.AbstractEpollChannel.doReadBytes(AbstractEpollChannel.java:244)
        at io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:790)
        at io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:345)
        at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:253)
        at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:699)
        at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:137)
        at java.lang.Thread.run(Thread.java:745)
    java.io.IOException: readAddress() failed: Invalid argument
        at io.netty.channel.epoll.Native.newIOException(Native.java:129)
        at io.netty.channel.epoll.Native.ioResult(Native.java:145)
        at io.netty.channel.epoll.Native.readAddress(Native.java:286)
        at io.netty.channel.epoll.AbstractEpollChannel.doReadBytes(AbstractEpollChannel.java:244)
        at io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:790)
        at io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:345)
        at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:253)
        at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:699)
        at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:137)
        at java.lang.Thread.run(Thread.java:745)

... many more of the same
```

@rkapsi - Thanks. Will check. Can I take some of your code you posted and morph it into another form of unit test?

@Scottmitch yes, feel free.

The readAddress() exceptions are possibly a result of me adding a finally block to that test (because I've been turning it also into an unit test on my end).

```
} finally {
      server.close().syncUninterruptibly();
    }
```

@rkapsi - Thanks. I have a PR pending approval. I will ping you when I push it upstream and have you verify the changes.


Pull request review comments:

Patch that resolved the issue:
diff --git a/transport-native-epoll/src/main/java/io/netty/channel/epoll/AbstractEpollChannel.java b/transport-native-epoll/src/main/java/io/netty/channel/epoll/AbstractEpollChannel.java
index d3faad8d2a1..491e79dcaf8 100644
--- a/transport-native-epoll/src/main/java/io/netty/channel/epoll/AbstractEpollChannel.java
+++ b/transport-native-epoll/src/main/java/io/netty/channel/epoll/AbstractEpollChannel.java
@@ -311,6 +311,23 @@ protected abstract class AbstractEpollUnsafe extends AbstractUnsafe {
          */
         abstract void epollInReady();
 
+        /**
+         * Will schedule a {@link #epollInReady()} call on the event loop if necessary.
+         * @param edgeTriggered {@code true} if the channel is using ET mode. {@code false} otherwise.
+         */
+        final void checkResetEpollIn(boolean edgeTriggered) {
+            if (edgeTriggered && !isInputShutdown0()) {
+                // trigger a read again as there may be something left to read and because of epoll ET we
+                // will not get notified again until we read everything from the socket
+                eventLoop().execute(new OneTimeTask() {
+                    @Override
+                    public void run() {
+                        epollInReady();
+                    }
+                });
+            }
+        }
+
         /**
          * Called once EPOLLRDHUP event is ready to be processed
          */
diff --git a/transport-native-epoll/src/main/java/io/netty/channel/epoll/AbstractEpollServerChannel.java b/transport-native-epoll/src/main/java/io/netty/channel/epoll/AbstractEpollServerChannel.java
index eaeded0d16a..d277a441581 100644
--- a/transport-native-epoll/src/main/java/io/netty/channel/epoll/AbstractEpollServerChannel.java
+++ b/transport-native-epoll/src/main/java/io/netty/channel/epoll/AbstractEpollServerChannel.java
@@ -73,7 +73,7 @@ protected Object filterOutboundMessage(Object msg) throws Exception {
     abstract Channel newChildChannel(int fd, byte[] remote, int offset, int len) throws Exception;
 
     final class EpollServerSocketUnsafe extends AbstractEpollUnsafe {
-        // Will hold the remote address after accept(...) was sucesssful.
+        // Will hold the remote address after accept(...) was successful.
         // We need 24 bytes for the address as maximum + 1 byte for storing the length.
         // So use 26 bytes as it's a power of two.
         private final byte[] acceptedAddress = new byte[26];
@@ -117,16 +117,8 @@ void epollInReady() {
                         readPending = false;
                         allocHandle.incMessagesRead(1);
 
-                        try {
-                            int len = acceptedAddress[0];
-                            pipeline.fireChannelRead(newChildChannel(socketFd, acceptedAddress, 1, len));
-                        } catch (Throwable t) {
-                            if (edgeTriggered) { // We must keep reading if ET is enabled
-                                pipeline.fireExceptionCaught(t);
-                            } else {
-                                throw t;
-                            }
-                        }
+                        int len = acceptedAddress[0];
+                        pipeline.fireChannelRead(newChildChannel(socketFd, acceptedAddress, 1, len));
                     } while (allocHandle.continueReading());
                 } catch (Throwable t) {
                     exception = t;
@@ -136,6 +128,7 @@ void epollInReady() {
 
                 if (exception != null) {
                     pipeline.fireExceptionCaught(exception);
+                    checkResetEpollIn(edgeTriggered);
                 }
             } finally {
                 // Check if there is a readPending which was not processed yet.
diff --git a/transport-native-epoll/src/main/java/io/netty/channel/epoll/AbstractEpollStreamChannel.java b/transport-native-epoll/src/main/java/io/netty/channel/epoll/AbstractEpollStreamChannel.java
index 1747fb2305c..50fb2a5df4f 100644
--- a/transport-native-epoll/src/main/java/io/netty/channel/epoll/AbstractEpollStreamChannel.java
+++ b/transport-native-epoll/src/main/java/io/netty/channel/epoll/AbstractEpollStreamChannel.java
@@ -587,7 +587,7 @@ private void safeClosePipe(int pipe) {
     }
 
     class EpollStreamUnsafe extends AbstractEpollUnsafe {
-        private boolean handleReadException(ChannelPipeline pipeline, ByteBuf byteBuf, Throwable cause, boolean close) {
+        private void handleReadException(ChannelPipeline pipeline, ByteBuf byteBuf, Throwable cause, boolean close) {
             if (byteBuf != null) {
                 if (byteBuf.isReadable()) {
                     readPending = false;
@@ -601,9 +601,7 @@ private boolean handleReadException(ChannelPipeline pipeline, ByteBuf byteBuf, T
             pipeline.fireExceptionCaught(cause);
             if (close || cause instanceof IOException) {
                 shutdownInput();
-                return true;
             }
-            return false;
         }
 
         @Override
@@ -769,48 +767,35 @@ void epollInReady() {
             boolean close = false;
             try {
                 do {
-                    try {
-                        SpliceInTask spliceTask = spliceQueue.peek();
-                        if (spliceTask != null) {
-                            if (spliceTask.spliceIn(allocHandle)) {
-                                // We need to check if it is still active as if not we removed all SpliceTasks in
-                                // doClose(...)
-                                if (isActive()) {
-                                    spliceQueue.remove();
-                                }
-                                continue;
-                            } else {
-                                break;
+                    SpliceInTask spliceTask = spliceQueue.peek();
+                    if (spliceTask != null) {
+                        if (spliceTask.spliceIn(allocHandle)) {
+                            // We need to check if it is still active as if not we removed all SpliceTasks in
+                            // doClose(...)
+                            if (isActive()) {
+                                spliceQueue.remove();
                             }
-                        }
-
-                        // we use a direct buffer here as the native implementations only be able
-                        // to handle direct buffers.
-                        byteBuf = allocHandle.allocate(allocator);
-                        allocHandle.lastBytesRead(doReadBytes(byteBuf));
-                        if (allocHandle.lastBytesRead() <= 0) {
-                            // nothing was read, release the buffer.
-                            byteBuf.release();
-                            byteBuf = null;
-                            close = allocHandle.lastBytesRead() < 0;
+                            continue;
+                        } else {
                             break;
                         }
-                        readPending = false;
-                        allocHandle.incMessagesRead(1);
-                        pipeline.fireChannelRead(byteBuf);
+                    }
+
+                    // we use a direct buffer here as the native implementations only be able
+                    // to handle direct buffers.
+                    byteBuf = allocHandle.allocate(allocator);
+                    allocHandle.lastBytesRead(doReadBytes(byteBuf));
+                    if (allocHandle.lastBytesRead() <= 0) {
+                        // nothing was read, release the buffer.
+                        byteBuf.release();
                         byteBuf = null;
-                    } catch (Throwable t) {
-                        if (edgeTriggered) { // We must keep reading if ET is enabled
-                            if (byteBuf != null) {
-                                byteBuf.release();
-                                byteBuf = null;
-                            }
-                            pipeline.fireExceptionCaught(t);
-                        } else {
-                            // byteBuf is release in outer exception handling if necessary.
-                            throw t;
-                        }
+                        close = allocHandle.lastBytesRead() < 0;
+                        break;
                     }
+                    readPending = false;
+                    allocHandle.incMessagesRead(1);
+                    pipeline.fireChannelRead(byteBuf);
+                    byteBuf = null;
                 } while (allocHandle.continueReading());
 
                 allocHandle.readComplete();
@@ -821,17 +806,8 @@ void epollInReady() {
                     close = false;
                 }
             } catch (Throwable t) {
-                boolean closed = handleReadException(pipeline, byteBuf, t, close);
-                if (!closed) {
-                    // trigger a read again as there may be something left to read and because of epoll ET we
-                    // will not get notified again until we read everything from the socket
-                    eventLoop().execute(new OneTimeTask() {
-                        @Override
-                        public void run() {
-                            epollInReady();
-                        }
-                    });
-                }
+                handleReadException(pipeline, byteBuf, t, close);
+                checkResetEpollIn(edgeTriggered);
             } finally {
                 // Check if there is a readPending which was not processed yet.
                 // This could be for two reasons:
diff --git a/transport-native-epoll/src/main/java/io/netty/channel/epoll/EpollDatagramChannel.java b/transport-native-epoll/src/main/java/io/netty/channel/epoll/EpollDatagramChannel.java
index 3681024e3a7..a9c5e976cf4 100644
--- a/transport-native-epoll/src/main/java/io/netty/channel/epoll/EpollDatagramChannel.java
+++ b/transport-native-epoll/src/main/java/io/netty/channel/epoll/EpollDatagramChannel.java
@@ -535,9 +535,9 @@ void epollInReady() {
 
             Throwable exception = null;
             try {
-                do {
-                    ByteBuf data = null;
-                    try {
+                ByteBuf data = null;
+                try {
+                    do {
                         data = allocHandle.allocate(allocator);
                         allocHandle.attemptedBytesRead(data.writableBytes());
                         final DatagramSocketAddress remoteAddress;
@@ -564,21 +564,14 @@ void epollInReady() {
 
                         readBuf.add(new DatagramPacket(data, (InetSocketAddress) localAddress(), remoteAddress));
                         data = null;
-                    } catch (Throwable t) {
-                        if (data != null) {
-                            data.release();
-                            data = null;
-                        }
-                        if (edgeTriggered) {
-                            // We do not break from the loop here and remember the last exception,
-                            // because we need to consume everything from the socket used with epoll ET.
-                            pipeline.fireExceptionCaught(t);
-                        } else {
-                            exception = t;
-                            break;
-                        }
+                    } while (allocHandle.continueReading());
+                } catch (Throwable t) {
+                    if (data != null) {
+                        data.release();
+                        data = null;
                     }
-                } while (allocHandle.continueReading());
+                    exception = t;
+                }
 
                 int size = readBuf.size();
                 for (int i = 0; i < size; i ++) {
@@ -590,6 +583,7 @@ void epollInReady() {
 
                 if (exception != null) {
                     pipeline.fireExceptionCaught(exception);
+                    checkResetEpollIn(edgeTriggered);
                 }
             } finally {
                 // Check if there is a readPending which was not processed yet.
diff --git a/transport-native-epoll/src/main/java/io/netty/channel/epoll/EpollDomainSocketChannel.java b/transport-native-epoll/src/main/java/io/netty/channel/epoll/EpollDomainSocketChannel.java
index fb94e2a85bb..63c130dc03e 100644
--- a/transport-native-epoll/src/main/java/io/netty/channel/epoll/EpollDomainSocketChannel.java
+++ b/transport-native-epoll/src/main/java/io/netty/channel/epoll/EpollDomainSocketChannel.java
@@ -157,16 +157,7 @@ private void epollInReadFd() {
 
                     readPending = false;
                     allocHandle.incMessagesRead(1);
-                    try {
-                        pipeline.fireChannelRead(new FileDescriptor(socketFd));
-                    } catch (Throwable t) {
-                        // If ET is enabled we need to consume everything from the socket
-                        if (edgeTriggered) {
-                            pipeline.fireExceptionCaught(t);
-                        } else {
-                            throw t;
-                        }
-                    }
+                    pipeline.fireChannelRead(new FileDescriptor(socketFd));
                 } while (allocHandle.continueReading());
 
                 allocHandle.readComplete();
@@ -175,14 +166,7 @@ private void epollInReadFd() {
                 allocHandle.readComplete();
                 pipeline.fireChannelReadComplete();
                 pipeline.fireExceptionCaught(t);
-                // trigger a read again as there may be something left to read and because of epoll ET we
-                // will not get notified again until we read everything from the socket
-                eventLoop().execute(new OneTimeTask() {
-                    @Override
-                    public void run() {
-                        epollInReady();
-                    }
-                });
+                checkResetEpollIn(edgeTriggered);
             } finally {
                 // Check if there is a readPending which was not processed yet.
                 // This could be for two reasons:


Test patch:
diff --git a/transport-native-epoll/src/test/java/io/netty/channel/epoll/EpollSocketChannelTest.java b/transport-native-epoll/src/test/java/io/netty/channel/epoll/EpollSocketChannelTest.java
index 9637006217e..c13ac8f7ee6 100644
--- a/transport-native-epoll/src/test/java/io/netty/channel/epoll/EpollSocketChannelTest.java
+++ b/transport-native-epoll/src/test/java/io/netty/channel/epoll/EpollSocketChannelTest.java
@@ -16,12 +16,27 @@
 package io.netty.channel.epoll;
 
 import io.netty.bootstrap.Bootstrap;
+import io.netty.bootstrap.ServerBootstrap;
+import io.netty.buffer.Unpooled;
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelHandlerContext;
 import io.netty.channel.ChannelInboundHandlerAdapter;
+import io.netty.channel.ChannelInitializer;
+import io.netty.channel.ChannelOption;
+import io.netty.channel.ChannelPipeline;
 import io.netty.channel.EventLoopGroup;
+import io.netty.channel.ServerChannel;
 import org.junit.Assert;
 import org.junit.Test;
 
 import java.net.InetSocketAddress;
+import java.net.SocketAddress;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
 
 public class EpollSocketChannelTest {
 
@@ -98,4 +113,97 @@ private static void assertTcpInfo0(EpollTcpInfo info) throws Exception {
         Assert.assertTrue(info.rcvSpace() >= 0);
         Assert.assertTrue(info.totalRetrans() >= 0);
     }
+
+    @Test
+    public void testExceptionHandlingDoesNotInfiniteLoop() throws InterruptedException {
+        EventLoopGroup group = new EpollEventLoopGroup();
+        try {
+            runExceptionHandleFeedbackLoop(group, EpollServerSocketChannel.class, EpollSocketChannel.class,
+                    new InetSocketAddress(0));
+            runExceptionHandleFeedbackLoop(group, EpollServerDomainSocketChannel.class, EpollDomainSocketChannel.class,
+                    EpollSocketTestPermutation.newSocketAddress());
+        } finally {
+            group.shutdownGracefully();
+        }
+    }
+
+    private void runExceptionHandleFeedbackLoop(EventLoopGroup group, Class<? extends ServerChannel> serverChannelClass,
+            Class<? extends Channel> channelClass, SocketAddress bindAddr) throws InterruptedException {
+        Channel serverChannel = null;
+        Channel clientChannel = null;
+        try {
+            MyInitializer serverInitializer = new MyInitializer();
+            ServerBootstrap sb = new ServerBootstrap();
+            sb.option(ChannelOption.SO_BACKLOG, 1024);
+            sb.group(group)
+            .channel(serverChannelClass)
+            .childHandler(serverInitializer);
+
+            serverChannel = sb.bind(bindAddr).syncUninterruptibly().channel();
+
+            Bootstrap b = new Bootstrap();
+            b.group(group);
+            b.channel(channelClass);
+            b.option(ChannelOption.SO_KEEPALIVE, true);
+            b.remoteAddress(serverChannel.localAddress());
+            b.handler(new MyInitializer());
+            clientChannel = b.connect().syncUninterruptibly().channel();
+
+            clientChannel.writeAndFlush(Unpooled.wrappedBuffer(new byte[1024]));
+
+            // We expect to get 2 exceptions (1 from BuggyChannelHandler and 1 from ExceptionHandler).
+            assertTrue(serverInitializer.exceptionHandler.latch1.await(2, TimeUnit.SECONDS));
+
+            // After we get the first exception, we should get no more, this is expected to timeout.
+            assertFalse("Encountered " + serverInitializer.exceptionHandler.count.get() +
+                    " exceptions when 1 was expected",
+                    serverInitializer.exceptionHandler.latch2.await(2, TimeUnit.SECONDS));
+        } finally {
+            if (serverChannel != null) {
+                serverChannel.close().syncUninterruptibly();
+            }
+            if (clientChannel != null) {
+                clientChannel.close().syncUninterruptibly();
+            }
+        }
+    }
+
+    private static class MyInitializer extends ChannelInitializer<Channel> {
+        final ExceptionHandler exceptionHandler = new ExceptionHandler();
+        @Override
+        protected void initChannel(Channel ch) throws Exception {
+            ChannelPipeline pipeline = ch.pipeline();
+
+            pipeline.addLast(new BuggyChannelHandler());
+            pipeline.addLast(exceptionHandler);
+        }
+    }
+
+    private static class BuggyChannelHandler extends ChannelInboundHandlerAdapter {
+        @Override
+        public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
+            throw new NullPointerException("I am a bug!");
+        }
+    }
+
+    private static class ExceptionHandler extends ChannelInboundHandlerAdapter {
+        final AtomicLong count = new AtomicLong();
+        /**
+         * We expect to get 2 calls to {@link #exceptionCaught(ChannelHandlerContext, Throwable)}.
+         * 1 call from BuggyChannelHandler and 1 from closing the channel in this class.
+         */
+        final CountDownLatch latch1 = new CountDownLatch(2);
+        final CountDownLatch latch2 = new CountDownLatch(1);
+
+        @Override
+        public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
+            if (count.incrementAndGet() <= 2) {
+                latch1.countDown();
+            } else {
+                latch2.countDown();
+            }
+            // This is expected to throw an exception!
+            ctx.close();
+        }
+    }
 }