Repository: hazelcast/hazelcast-simulator
Issue: Stabilizer not Failing,  error go unreported

Due to a simple error in test code I found this in worker.log

```
grepall worker.log iCacheCas
./worker-10.184.184.149-1-server/worker.log
INFO  2014-09-11 12:48:24,900 [Thread-0] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Initializing test iCacheCas ---------------------------
      id=iCacheCas
INFO  2014-09-11 12:48:26,081 [Thread-2] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Starting iCacheCas.setup() ------------------------------------
FATAL 2014-09-11 12:48:26,081 [Thread-2] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Failed iCacheCas.setup() ---------------------------
INFO  2014-09-11 12:48:28,098 [Thread-3] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Starting iCacheCas.localWarmup() ------------------------------------
INFO  2014-09-11 12:48:28,098 [Thread-3] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Finished iCacheCas.localWarmup() ---------------------------
INFO  2014-09-11 12:48:32,115 [Thread-4] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Skipping iCacheCas.run(); member is passive ------------------------------------
INFO  2014-09-11 12:50:33,277 [Thread-0] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- iCacheCas.stop() ------------------------------------
INFO  2014-09-11 12:50:36,310 [Thread-5] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Starting iCacheCas.localVerify() ------------------------------------
INFO  2014-09-11 12:50:36,310 [Thread-5] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Finished iCacheCas.localVerify() ---------------------------
INFO  2014-09-11 12:50:41,327 [Thread-6] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Starting iCacheCas.localTeardown() ------------------------------------
FATAL 2014-09-11 12:50:41,327 [Thread-6] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Failed iCacheCas.localTeardown() ---------------------------

./worker-10.67.172.55-2-client/worker.log
INFO  2014-09-11 12:48:25,439 [Thread-0] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Initializing test iCacheCas ---------------------------
      id=iCacheCas
INFO  2014-09-11 12:48:26,623 [Thread-2] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Starting iCacheCas.setup() ------------------------------------
FATAL 2014-09-11 12:48:26,715 [Thread-2] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Failed iCacheCas.setup() ---------------------------
INFO  2014-09-11 12:48:28,701 [Thread-3] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Starting iCacheCas.localWarmup() ------------------------------------
INFO  2014-09-11 12:48:28,702 [Thread-3] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Finished iCacheCas.localWarmup() ---------------------------
INFO  2014-09-11 12:48:32,717 [Thread-4] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Starting iCacheCas.run() ------------------------------------
FATAL 2014-09-11 12:48:32,717 [Thread-4] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Failed to execute iCacheCas.run() ------------------------------------
INFO  2014-09-11 12:50:33,875 [Thread-0] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- iCacheCas.stop() ------------------------------------
INFO  2014-09-11 12:50:36,889 [Thread-5] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Starting iCacheCas.localVerify() ------------------------------------
INFO  2014-09-11 12:50:36,889 [Thread-5] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Finished iCacheCas.localVerify() ---------------------------
INFO  2014-09-11 12:50:41,916 [Thread-6] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Starting iCacheCas.localTeardown() ------------------------------------
FATAL 2014-09-11 12:50:41,916 [Thread-6] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Failed iCacheCas.localTeardown() ---------------------------

./worker-10.184.184.149-2-client/worker.log
INFO  2014-09-11 12:48:25,409 [Thread-0] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Initializing test iCacheCas ---------------------------
      id=iCacheCas
INFO  2014-09-11 12:48:26,578 [Thread-2] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Starting iCacheCas.setup() ------------------------------------
FATAL 2014-09-11 12:48:26,689 [Thread-2] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Failed iCacheCas.setup() ---------------------------
INFO  2014-09-11 12:48:28,590 [Thread-3] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Starting iCacheCas.localWarmup() ------------------------------------
INFO  2014-09-11 12:48:28,590 [Thread-3] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Finished iCacheCas.localWarmup() ---------------------------
INFO  2014-09-11 12:48:32,608 [Thread-4] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Starting iCacheCas.run() ------------------------------------
FATAL 2014-09-11 12:48:32,609 [Thread-4] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Failed to execute iCacheCas.run() ------------------------------------
INFO  2014-09-11 12:50:33,769 [Thread-0] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- iCacheCas.stop() ------------------------------------
INFO  2014-09-11 12:50:36,795 [Thread-5] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Starting iCacheCas.localVerify() ------------------------------------
INFO  2014-09-11 12:50:36,795 [Thread-5] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Finished iCacheCas.localVerify() ---------------------------
INFO  2014-09-11 12:50:41,821 [Thread-6] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Starting iCacheCas.localTeardown() ------------------------------------
FATAL 2014-09-11 12:50:41,821 [Thread-6] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Failed iCacheCas.localTeardown() ---------------------------

./worker-10.67.172.55-1-server/worker.log
INFO  2014-09-11 12:48:25,857 [Thread-0] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Initializing test iCacheCas ---------------------------
      id=iCacheCas
INFO  2014-09-11 12:48:27,000 [Thread-2] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Starting iCacheCas.setup() ------------------------------------
FATAL 2014-09-11 12:48:27,000 [Thread-2] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Failed iCacheCas.setup() ---------------------------
INFO  2014-09-11 12:48:29,020 [Thread-3] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Starting iCacheCas.localWarmup() ------------------------------------
INFO  2014-09-11 12:48:29,020 [Thread-3] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Finished iCacheCas.localWarmup() ---------------------------
INFO  2014-09-11 12:48:31,028 [Thread-4] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Starting iCacheCas.globalWarmup() ------------------------------------
FATAL 2014-09-11 12:48:31,044 [Thread-4] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Failed iCacheCas.globalWarmup() ---------------------------
INFO  2014-09-11 12:48:33,037 [Thread-5] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Skipping iCacheCas.run(); member is passive ------------------------------------
INFO  2014-09-11 12:50:33,199 [Thread-0] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- iCacheCas.stop() ------------------------------------
INFO  2014-09-11 12:50:35,213 [Thread-6] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Starting iCacheCas.globalVerify() ------------------------------------
FATAL 2014-09-11 12:50:35,213 [Thread-6] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Failed iCacheCas.globalVerify() ---------------------------
INFO  2014-09-11 12:50:37,223 [Thread-7] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Starting iCacheCas.localVerify() ------------------------------------
INFO  2014-09-11 12:50:37,224 [Thread-7] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Finished iCacheCas.localVerify() ---------------------------
INFO  2014-09-11 12:50:39,235 [Thread-8] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Starting iCacheCas.globalTeardown() ------------------------------------
INFO  2014-09-11 12:50:39,235 [Thread-8] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Finished iCacheCas.globalTeardown() ---------------------------
INFO  2014-09-11 12:50:42,266 [Thread-9] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Starting iCacheCas.localTeardown() ------------------------------------
FATAL 2014-09-11 12:50:42,266 [Thread-9] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Failed iCacheCas.localTeardown() ---------------------------
```

While Stabilizer reported 

```
INFO  12:48:24 iCacheCas iCacheCas Starting Test initialization
INFO  12:48:25 iCacheCas Completed Test initialization
INFO  12:48:25 iCacheCas Starting Test setup
INFO  12:48:27 iCacheCas Completed Test setup
INFO  12:48:28 iCacheCas Starting Test local warmup
INFO  12:48:30 iCacheCas Completed Test local warmup
INFO  12:48:30 iCacheCas Starting Test global warmup
INFO  12:48:32 iCacheCas Completed Test global warmup
INFO  12:48:32 iCacheCas Starting Test start
INFO  12:48:33 iCacheCas Completed Test start
INFO  12:48:33 iCacheCas Test will run for 00d 00h 02m 00s
INFO  12:49:03 iCacheCas Running 00d 00h 00m 30s, 25.00 percent complete
INFO  12:49:33 iCacheCas Running 00d 00h 01m 00s, 50.00 percent complete
INFO  12:50:03 iCacheCas Running 00d 00h 01m 30s, 75.00 percent complete
INFO  12:50:33 iCacheCas Running 00d 00h 02m 00s, 100.00 percent complete
INFO  12:50:33 iCacheCas Test finished running
INFO  12:50:33 iCacheCas Starting Test stop
INFO  12:50:34 iCacheCas Completed Test stop
INFO  12:50:34 iCacheCas Starting Test global verify
INFO  12:50:36 iCacheCas Completed Test global verify
INFO  12:50:36 iCacheCas Starting Test local verify
INFO  12:50:38 iCacheCas Completed Test local verify
INFO  12:50:38 iCacheCas Starting Test global tear down
INFO  12:50:40 iCacheCas Finished Test global tear down
INFO  12:50:40 iCacheCas Starting Test local tear down
INFO  12:50:42 iCacheCas Completed Test local tear down
INFO  12:50:42 Terminating workers
INFO  12:50:42 All workers have been terminated
INFO  12:50:42 Starting cool down (10 sec)
INFO  12:50:52 Finished cool down
INFO  12:50:52 Total running time: 147 seconds
INFO  12:50:52 -----------------------------------------------------------------------------
INFO  12:50:52 No failures have been detected!
INFO  12:50:52 -----------------------------------------------------------------------------
```

details of setup error

```
due to a simple error in my test setup method,  I found this error in worker.log

```

FATAL 2014-09-11 12:48:26,081 [Thread-2] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Failed iCacheCas.setup() ---------------------------
FATAL 2014-09-11 12:48:26,082 [Thread-2] com.hazelcast.stabilizer.tests.utils.ExceptionReporter: Exception detected
java.lang.NullPointerException: Retrieving a list instance with a null name is not allowed!
    at com.hazelcast.instance.HazelcastInstanceImpl.getList(HazelcastInstanceImpl.java:208)
    at com.hazelcast.instance.HazelcastInstanceProxy.getList(HazelcastInstanceProxy.java:91)
    at com.hazelcast.stabilizer.tests.icache.CasICacheTest.setup(CasICacheTest.java:66)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:606)
    at com.hazelcast.stabilizer.worker.TestContainer.invoke(TestContainer.java:138)
    at com.hazelcast.stabilizer.worker.TestContainer.setup(TestContainer.java:101)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:606)
    at com.hazelcast.stabilizer.worker.MemberWorker$TestCommandRequestProcessingThread$2.doRun(MemberWorker.java:405)
    at com.hazelcast.stabilizer.worker.MemberWorker$CommandThread.run(MemberWorker.java:493)

```


And due to a simple error in localTeardown

```

FATAL 2014-09-11 12:50:41,327 [Thread-6] com.hazelcast.stabilizer.worker.MemberWorker: --------------------------- Failed iCacheCas.localTeardown() ---------------------------
FATAL 2014-09-11 12:50:41,327 [Thread-6] com.hazelcast.stabilizer.tests.utils.ExceptionReporter: Exception detected
java.lang.NullPointerException
    at com.hazelcast.stabilizer.tests.icache.CasICacheTest.teardown(CasICacheTest.java:82)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:606)
    at com.hazelcast.stabilizer.worker.TestContainer.invoke(TestContainer.java:138)
    at com.hazelcast.stabilizer.worker.TestContainer.localTeardown(TestContainer.java:109)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:606)
    at com.hazelcast.stabilizer.worker.MemberWorker$TestCommandRequestProcessingThread$2.doRun(MemberWorker.java:405)
    at com.hazelcast.stabilizer.worker.MemberWorker$CommandThread.run(MemberWorker.java:493)

```
```


Changed files:
stabilizer/src/main/java/com/hazelcast/stabilizer/tests/utils/ExceptionReporter.java
stabilizer/src/main/java/com/hazelcast/stabilizer/worker/MemberWorker.java

diff --git a/stabilizer/src/main/java/com/hazelcast/stabilizer/tests/utils/ExceptionReporter.java b/stabilizer/src/main/java/com/hazelcast/stabilizer/tests/utils/ExceptionReporter.java
index 1f92e2fa1..f8efb749b 100644
--- a/stabilizer/src/main/java/com/hazelcast/stabilizer/tests/utils/ExceptionReporter.java
+++ b/stabilizer/src/main/java/com/hazelcast/stabilizer/tests/utils/ExceptionReporter.java
@@ -15,7 +15,7 @@ import static com.hazelcast.stabilizer.Utils.writeText;
  */
 public class ExceptionReporter {
 
-    private final static AtomicLong FAILURE_ID = new AtomicLong(1);
+    private final static AtomicLong FAILURE_ID = new AtomicLong(0);
     private final static ILogger log = Logger.getLogger(ExceptionReporter.class);
 
     /**
@@ -31,16 +31,27 @@ public class ExceptionReporter {
             return;
         }
 
-        log.severe("Exception detected", cause);
+        long exceptionCount = FAILURE_ID.incrementAndGet();
+        log.severe("Exception #" + exceptionCount + " detected", cause);
 
-        final File tmpFile = createTmpFile();
-        if (tmpFile == null) {
+        String targetFileName = exceptionCount + ".exception";
+
+        final File tmpFile = new File(targetFileName + ".tmp");
+
+        try {
+            if (!tmpFile.createNewFile()) {
+                // can't happen since id's are always incrementing. So just for safety reason this is added.
+                throw new IOException("Could not create tmp file:" + tmpFile.getAbsolutePath()+" file already exists.");
+            }
+        } catch (IOException e) {
+            log.severe("Could not report exception; this means that this exception is not visible to the coordinator", e);
             return;
         }
 
         writeCauseToFile(testId, cause, tmpFile);
 
-        final File file = new File(FAILURE_ID.incrementAndGet() + ".exception");
+        final File file = new File(targetFileName);
+
         if (!tmpFile.renameTo(file)) {
             log.severe("Failed to rename tmp file:" + tmpFile + " to " + file);
         }
diff --git a/stabilizer/src/main/java/com/hazelcast/stabilizer/worker/MemberWorker.java b/stabilizer/src/main/java/com/hazelcast/stabilizer/worker/MemberWorker.java
index 6b0c3f86b..4c62ac522 100644
--- a/stabilizer/src/main/java/com/hazelcast/stabilizer/worker/MemberWorker.java
+++ b/stabilizer/src/main/java/com/hazelcast/stabilizer/worker/MemberWorker.java
@@ -346,7 +346,7 @@ public class MemberWorker {
                     return;
                 }
 
-                new CommandThread(command, testId) {
+                CommandThread commandThread = new CommandThread(command, testId) {
                     @Override
                     public void doRun() throws Throwable {
                         boolean passive = command.clientOnly && clientInstance == null;
@@ -365,15 +365,16 @@ public class MemberWorker {
                                 log.info(format("--------------------------- Completed %s.run() " +
                                                 "------------------------------------",
                                         testName));
-                            }catch(Throwable t){
+                            } catch (Throwable t) {
                                 String msg = format("--------------------------- Failed to execute %s.run() " +
                                                 "------------------------------------",
                                         testName);
-                                log.severe(msg,t);
+                                log.severe(msg, t);
                             }
                         }
                     }
-                }.start();
+                };
+                commandThread.start();
             } catch (Exception e) {
                 log.severe("Failed to start test", e);
                 throw e;
@@ -395,7 +396,7 @@ public class MemberWorker {
                 }
 
                 final Method method = test.getClass().getMethod(methodName);
-                new CommandThread(command, command.testId) {
+                CommandThread commandThread = new CommandThread(command, command.testId) {
                     @Override
                     public void doRun() throws Throwable {
                         log.info(format("--------------------------- Starting %s.%s() ------------------------------------",
@@ -415,7 +416,8 @@ public class MemberWorker {
                             }
                         }
                     }
-                }.start();
+                };
+                commandThread.start();
             } catch (Exception e) {
                 log.severe(format("Failed to execute test.%s()", methodName), e);
                 throw e;
@@ -492,7 +494,7 @@ public class MemberWorker {
                 commands.put(testId, command);
                 doRun();
             } catch (Throwable t) {
-                ExceptionReporter.report(null, t);
+                ExceptionReporter.report(testId, t);
             } finally {
                 commands.remove(testId);
             }