Repository: netty/netty
Issue: UnorderedThreadPoolEventExecutor consumes 100% CPU when idle
Labels: defect

### Expected behavior
0% CPU consumption when idle
### Actual behavior
almost 100% CPU consumption when idle
### Steps to reproduce
After a submitted task has been successfully executed, all the available cores get busy with doing some stuff.
```java
import io.netty.util.concurrent.UnorderedThreadPoolEventExecutor;

public class TestUnorderedThreadPoolEventExecutor {

    public static void main(String[] args) {
        UnorderedThreadPoolEventExecutor executor = new UnorderedThreadPoolEventExecutor(1);
        // compare: ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(1);
        executor.submit(() -> {
            return "completed!";
        });
    }
}
```
### Netty version
```
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
<version>4.1.8.Final</version>
```
### OS and JVM versions
#### Linux
Linux 3.13.0-042stab120.3 x86_64 x86_64 x86_64 GNU/Linux
java version "1.8.0_112"
Java(TM) SE Runtime Environment (build 1.8.0_112-b15)
Java HotSpot(TM) 64-Bit Server VM (build 25.112-b15, mixed mode)
#### OSX
Darwin 15.6.0 Darwin Kernel Version 15.6.0: Mon Jan  9 23:07:29 PST 2017; root:xnu-3248.60.11.2.1~1/RELEASE_X86_64 x86_64
java version "1.8.0_121"
Java(TM) SE Runtime Environment (build 1.8.0_121-b13)
Java HotSpot(TM) 64-Bit Server VM (build 25.121-b13, mixed mode)

### `top -H` on Linux
```
top - 13:37:31 up 92 days, 47 min,  2 users,  load average: 0.57, 0.13, 0.04
Threads: 127 total,   3 running, 124 sleeping,   0 stopped,   0 zombie
%Cpu(s): 96.4 us,  0.5 sy,  0.0 ni,  3.1 id,  0.0 wa,  0.0 hi,  0.0 si,  0.0 st
KiB Mem:   2097152 total,  2082604 used,    14548 free,        0 buffers
KiB Swap:        0 total,        0 used,        0 free.  1245400 cached Mem

  PID USER      PR  NI    VIRT    RES    SHR S %CPU %MEM     TIME+ COMMAND                                                                                                                                
28999 miha      20   0 2716608 599284  13700 R 96.6 28.6   0:13.76 java                                                                                                                                  
28998 miha      20   0 2716608 599284  13700 R 96.3 28.6   0:13.78 java
29000 miha      20   0 2716608 599284  13700 S  0.7 28.6   0:00.07 java                                                                                                                                   
29011 miha      20   0   21568   1640   1136 R  0.3  0.1   0:00.01 top 
...     
```
### top and jstack on OSX
```
$ top
Processes: 341 total, 3 running, 12 stuck, 326 sleeping, 1896 threads                                                                                                                             13:42:07
Load Avg: 5.01, 3.82, 3.51  CPU usage: 91.67% user, 2.97% sys, 5.35% idle  SharedLibs: 90M resident, 14M data, 4096B linkedit. MemRegions: 262828 total, 6832M resident, 79M private, 1222M shared.
PhysMem: 15G used (2782M wired), 1431M unused. VM: 977G vsize, 533M framework vsize, 33508915(64) swapins, 35619367(0) swapouts. Networks: packets: 37262432/90G in, 23652973/58G out.
Disks: 18020162/1162G read, 13501631/688G written.

PID    COMMAND      %CPU  TIME     #TH   #WQ  #PORTS MEM    PURG   CMPRS  PGRP  PPID  STATE    BOOSTS           %CPU_ME %CPU_OTHRS UID  FAULTS     COW       MSGSENT     MSGRECV     SYSBSD      SYSMACH
94811  top          4.7   00:00.96 1/1   0    21     6104K+ 0B     0B     94811 702   running  *0[1]            0.00000 0.00000    0    4993+      102       629591+     314787+     12608+      341476+
94809  java         727.4 01:24.36 23/8  0    87     1511M+ 0B     0B     94809 687   running  *0[2]            0.00000 0.00000    501  392525+    452       284         109         18048+      599646+
....

$ jstack 94809
2017-03-07 13:42:53
Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.121-b13 mixed mode):

"Attach Listener" #13 daemon prio=9 os_prio=31 tid=0x00007f8ebd000000 nid=0x3d0b waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"DestroyJavaVM" #12 prio=5 os_prio=31 tid=0x00007f8eb5802000 nid=0xc07 waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"unorderedThreadPoolEventExecutor-1-1" #11 prio=5 os_prio=31 tid=0x00007f8eb592a800 nid=0x5b0f runnable [0x0000700001658000]
   java.lang.Thread.State: RUNNABLE
	at java.util.concurrent.ScheduledThreadPoolExecutor.triggerTime(ScheduledThreadPoolExecutor.java:493)
	at java.util.concurrent.ScheduledThreadPoolExecutor.schedule(ScheduledThreadPoolExecutor.java:532)
	at io.netty.util.concurrent.UnorderedThreadPoolEventExecutor.schedule(UnorderedThreadPoolEventExecutor.java:171)
	at io.netty.util.concurrent.UnorderedThreadPoolEventExecutor.schedule(UnorderedThreadPoolEventExecutor.java:40)
	at java.util.concurrent.ScheduledThreadPoolExecutor.execute(ScheduledThreadPoolExecutor.java:622)
	at io.netty.util.concurrent.DefaultPromise.safeExecute(DefaultPromise.java:760)
	at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:428)
	at io.netty.util.concurrent.DefaultPromise.setSuccess(DefaultPromise.java:95)
	at io.netty.util.concurrent.PromiseTask.setSuccessInternal(PromiseTask.java:106)
	at io.netty.util.concurrent.PromiseTask.run(PromiseTask.java:74)
	at io.netty.util.concurrent.UnorderedThreadPoolEventExecutor$RunnableScheduledFutureTask.run(UnorderedThreadPoolEventExecutor.java:223)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:144)
	at java.lang.Thread.run(Thread.java:745)

"Service Thread" #9 daemon prio=9 os_prio=31 tid=0x00007f8eb5812000 nid=0x5303 runnable [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"C1 CompilerThread3" #8 daemon prio=9 os_prio=31 tid=0x00007f8eb5811000 nid=0x5103 waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"C2 CompilerThread2" #7 daemon prio=9 os_prio=31 tid=0x00007f8eb4803800 nid=0x4f03 waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"C2 CompilerThread1" #6 daemon prio=9 os_prio=31 tid=0x00007f8eb5801000 nid=0x4d03 waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"C2 CompilerThread0" #5 daemon prio=9 os_prio=31 tid=0x00007f8eb3056000 nid=0x4b03 waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"Signal Dispatcher" #4 daemon prio=9 os_prio=31 tid=0x00007f8eb3055800 nid=0x3e0f runnable [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"Finalizer" #3 daemon prio=8 os_prio=31 tid=0x00007f8eb405b000 nid=0x3803 in Object.wait() [0x0000700000d3a000]
   java.lang.Thread.State: WAITING (on object monitor)
	at java.lang.Object.wait(Native Method)
	- waiting on <0x00000006c679b6b0> (a java.lang.ref.ReferenceQueue$Lock)
	at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143)
	- locked <0x00000006c679b6b0> (a java.lang.ref.ReferenceQueue$Lock)
	at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164)
	at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209)

"Reference Handler" #2 daemon prio=10 os_prio=31 tid=0x00007f8eb4058000 nid=0x3603 in Object.wait() [0x0000700000c37000]
   java.lang.Thread.State: WAITING (on object monitor)
	at java.lang.Object.wait(Native Method)
	- waiting on <0x00000006c671fd30> (a java.lang.ref.Reference$Lock)
	at java.lang.Object.wait(Object.java:502)
	at java.lang.ref.Reference.tryHandlePending(Reference.java:191)
	- locked <0x00000006c671fd30> (a java.lang.ref.Reference$Lock)
	at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153)

"VM Thread" os_prio=31 tid=0x00007f8eb4053800 nid=0x3403 runnable 

"GC task thread#0 (ParallelGC)" os_prio=31 tid=0x00007f8eb580e000 nid=0x1707 runnable 

"GC task thread#1 (ParallelGC)" os_prio=31 tid=0x00007f8eb580e800 nid=0x2603 runnable 

"GC task thread#2 (ParallelGC)" os_prio=31 tid=0x00007f8eb580f800 nid=0x2803 runnable 

"GC task thread#3 (ParallelGC)" os_prio=31 tid=0x00007f8eb5810000 nid=0x2a03 runnable 

"GC task thread#4 (ParallelGC)" os_prio=31 tid=0x00007f8eb5810800 nid=0x2c03 runnable 

"GC task thread#5 (ParallelGC)" os_prio=31 tid=0x00007f8eb3005800 nid=0x2e03 runnable 

"GC task thread#6 (ParallelGC)" os_prio=31 tid=0x00007f8eb3007000 nid=0x3003 runnable 

"GC task thread#7 (ParallelGC)" os_prio=31 tid=0x00007f8eb4002800 nid=0x3203 runnable 

"VM Periodic Task Thread" os_prio=31 tid=0x00007f8eb4028800 nid=0x5503 waiting on condition 

JNI global references: 249
```


Issue comments:
@meshcow will have a look soon
@meshcow PTAL https://github.com/netty/netty/pull/6518 
looks good now!
Fixed by https://github.com/netty/netty/pull/6518

Pull request review comments:

Patch that resolved the issue:
diff --git a/common/src/main/java/io/netty/util/concurrent/UnorderedThreadPoolEventExecutor.java b/common/src/main/java/io/netty/util/concurrent/UnorderedThreadPoolEventExecutor.java
index 5d073debec3..4ed94da5375 100644
--- a/common/src/main/java/io/netty/util/concurrent/UnorderedThreadPoolEventExecutor.java
+++ b/common/src/main/java/io/netty/util/concurrent/UnorderedThreadPoolEventExecutor.java
@@ -30,6 +30,8 @@
 import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.TimeUnit;
 
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
+
 /**
  * {@link EventExecutor} implementation which makes no guarantees about the ordering of task execution that
  * are submitted because there may be multiple threads executing these tasks.
@@ -158,7 +160,8 @@ public Iterator<EventExecutor> iterator() {
 
     @Override
     protected <V> RunnableScheduledFuture<V> decorateTask(Runnable runnable, RunnableScheduledFuture<V> task) {
-        return new RunnableScheduledFutureTask<V>(this, runnable, task);
+        return runnable instanceof NonNotifyRunnable ?
+                task : new RunnableScheduledFutureTask<V>(this, runnable, task);
     }
 
     @Override
@@ -201,6 +204,11 @@ public <T> Future<T> submit(Callable<T> task) {
         return (Future<T>) super.submit(task);
     }
 
+    @Override
+    public void execute(Runnable command) {
+        super.schedule(new NonNotifyRunnable(command), 0, NANOSECONDS);
+    }
+
     private static final class RunnableScheduledFutureTask<V> extends PromiseTask<V>
             implements RunnableScheduledFuture<V>, ScheduledFuture<V> {
         private final RunnableScheduledFuture<V> future;
@@ -248,4 +256,25 @@ public int compareTo(Delayed o) {
             return future.compareTo(o);
         }
     }
+
+    // This is a special wrapper which we will be used in execute(...) to wrap the submitted Runnable. This is needed as
+    // ScheduledThreadPoolExecutor.execute(...) will delegate to submit(...) which will then use decorateTask(...).
+    // The problem with this is that decorateTask(...) needs to ensure we only do our own decoration if we not call
+    // from execute(...) as otherwise we may end up creating an endless loop because DefaultPromise will call
+    // EventExecutor.execute(...) when notify the listeners of the promise.
+    //
+    // See https://github.com/netty/netty/issues/6507
+    private static final class NonNotifyRunnable implements Runnable {
+
+        private final Runnable task;
+
+        NonNotifyRunnable(Runnable task) {
+            this.task = task;
+        }
+
+        @Override
+        public void run() {
+            task.run();
+        }
+    }
 }


Test patch:
diff --git a/common/src/test/java/io/netty/util/concurrent/UnorderedThreadPoolEventExecutorTest.java b/common/src/test/java/io/netty/util/concurrent/UnorderedThreadPoolEventExecutorTest.java
new file mode 100644
index 00000000000..d96db3fcb07
--- /dev/null
+++ b/common/src/test/java/io/netty/util/concurrent/UnorderedThreadPoolEventExecutorTest.java
@@ -0,0 +1,57 @@
+/*
+ * Copyright 2017 The Netty Project
+ *
+ * The Netty Project licenses this file to you under the Apache License,
+ * version 2.0 (the "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at:
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package io.netty.util.concurrent;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.concurrent.CountDownLatch;
+
+public class UnorderedThreadPoolEventExecutorTest {
+
+    // See https://github.com/netty/netty/issues/6507
+    @Test
+    public void testNotEndlessExecute() throws Exception {
+        UnorderedThreadPoolEventExecutor executor = new UnorderedThreadPoolEventExecutor(1);
+
+        try {
+            final CountDownLatch latch = new CountDownLatch(3);
+            Runnable task = new Runnable() {
+                @Override
+                public void run() {
+                    latch.countDown();
+                }
+            };
+            executor.execute(task);
+            Future<?> future = executor.submit(task).addListener(new FutureListener<Object>() {
+                @Override
+                public void operationComplete(Future<Object> future) throws Exception {
+                    latch.countDown();
+                }
+            });
+            latch.await();
+            future.syncUninterruptibly();
+
+            // Now just check if the queue stays empty multiple times. This is needed as the submit to execute(...)
+            // by DefaultPromise may happen in an async fashion
+            for (int i = 0; i < 10000; i++) {
+                Assert.assertTrue(executor.getQueue().isEmpty());
+            }
+        } finally {
+            executor.shutdownGracefully();
+        }
+    }
+}