Repository: hazelcast/hazelcast-simulator
Issue: Problem starting members, doesn't lead to failure

When there is a problem starting members, this isn't translated to a nice and useful message

```
INFO  11:25:49 =========================================================================
INFO  11:25:49 Starting 2 Workers (2 members, 0 clients)...
FATAL 11:25:50 Could not create 1 member Worker on C_A1 (EXCEPTION_DURING_OPERATION_EXECUTION)
FATAL 11:25:50 [C_A2] Failed to start Worker: Failed to start Worker
FATAL 11:25:51 [C_A1] Failed to start Worker: Failed to start Worker
INFO  11:25:51 Terminating 0 Workers...
INFO  11:25:52 Waiting up to 120 seconds for shutdown of 0 Workers...
INFO  11:25:52 Finished shutdown of all Workers (0 seconds)
INFO  11:25:52 Shutdown of com.hazelcast.simulator.protocol.connector.CoordinatorConnector...
INFO  11:25:52 ServerConnectorMessageQueueThread received POISON_PILL and will stop...
INFO  11:25:52 HarakiriMonitor is enabled and will kill inactive EC2 instances after 7200 seconds
INFO  11:25:52 Stopping 2 Agents...
INFO  11:25:52 Stopping Agent 54.158.82.62
INFO  11:25:52 Stopping Agent 54.205.232.99
INFO  11:25:54 Starting HarakiriMonitor on 54.205.232.99
INFO  11:25:54 Starting HarakiriMonitor on 54.158.82.62
INFO  11:25:56 Successfully stopped 2 Agents
INFO  11:25:56 Download artifacts of 2 machines...
INFO  11:25:56 Downloading Worker logs from 54.205.232.99
INFO  11:25:56 Downloading Worker logs from 54.158.82.62
INFO  11:25:56 Downloading Agent logs from 54.158.82.62
INFO  11:25:56 Downloading Agent logs from 54.205.232.99
INFO  11:26:04 Finished downloading artifacts of 2 machines (7 seconds)
INFO  11:26:04 Executing after-completion script: /home/alarmnummer/hazelcast-simulator-0.9-RC1-SNAPSHOT/conf/after-completion.sh
INFO  11:26:04 Finished after-completion script
INFO  11:26:04 =========================================================================
INFO  11:26:04 No failures have been detected!
INFO  11:26:04 =========================================================================
```


Changed files:
simulator/src/main/java/com/hazelcast/simulator/common/FailureType.java
simulator/src/main/java/com/hazelcast/simulator/coordinator/FailureCollector.java
simulator/src/main/java/com/hazelcast/simulator/protocol/operation/FailureOperation.java
simulator/src/main/java/com/hazelcast/simulator/coordinator/CoordinatorRunMonolith.java

diff --git a/simulator/src/main/java/com/hazelcast/simulator/common/FailureType.java b/simulator/src/main/java/com/hazelcast/simulator/common/FailureType.java
index e3fb5f524..352c7024d 100644
--- a/simulator/src/main/java/com/hazelcast/simulator/common/FailureType.java
+++ b/simulator/src/main/java/com/hazelcast/simulator/common/FailureType.java
@@ -27,7 +27,8 @@ public enum FailureType {
     WORKER_TIMEOUT("workerTimeout", "Worker timeout", false),
     WORKER_OOME("workerOOME", "Worker OOME", true),
     WORKER_ABNORMAL_EXIT("workerAbnormalExit", "Worker abnormal exit", true),
-    WORKER_NORMAL_EXIT("workerNormalExit", "Worker normal exit", true);
+    WORKER_NORMAL_EXIT("workerNormalExit", "Worker normal exit", true),
+    WORKER_CREATE_ERROR("workerCreateError", "Worker create error", true);
 
     private final String id;
     private final String humanReadable;
diff --git a/simulator/src/main/java/com/hazelcast/simulator/coordinator/CoordinatorRunMonolith.java b/simulator/src/main/java/com/hazelcast/simulator/coordinator/CoordinatorRunMonolith.java
index 04a48723d..878b56c48 100644
--- a/simulator/src/main/java/com/hazelcast/simulator/coordinator/CoordinatorRunMonolith.java
+++ b/simulator/src/main/java/com/hazelcast/simulator/coordinator/CoordinatorRunMonolith.java
@@ -16,7 +16,9 @@
 
 package com.hazelcast.simulator.coordinator;
 
+import com.hazelcast.simulator.common.FailureType;
 import com.hazelcast.simulator.common.TestPhase;
+import com.hazelcast.simulator.protocol.operation.FailureOperation;
 import com.hazelcast.simulator.utils.CommandLineExitException;
 import org.apache.log4j.Logger;
 
@@ -48,7 +50,13 @@ class CoordinatorRunMonolith {
     public void init(DeploymentPlan deploymentPlan) {
         logConfiguration(deploymentPlan);
 
-        coordinator.createStartWorkersTask(deploymentPlan.getWorkerDeployment(), new HashMap<String, String>()).run();
+        try {
+            coordinator.createStartWorkersTask(deploymentPlan.getWorkerDeployment(), new HashMap<String, String>()).run();
+        } catch (RuntimeException e) {
+            failureCollector.notify(
+                    new FailureOperation("Failed to create worker", FailureType.WORKER_CREATE_ERROR, null, null, null));
+            throw e;
+        }
     }
 
     public boolean run(TestSuite testSuite) throws Exception {
diff --git a/simulator/src/main/java/com/hazelcast/simulator/coordinator/FailureCollector.java b/simulator/src/main/java/com/hazelcast/simulator/coordinator/FailureCollector.java
index 83dacc0ca..8218a0548 100644
--- a/simulator/src/main/java/com/hazelcast/simulator/coordinator/FailureCollector.java
+++ b/simulator/src/main/java/com/hazelcast/simulator/coordinator/FailureCollector.java
@@ -15,6 +15,7 @@
  */
 package com.hazelcast.simulator.coordinator;
 
+import com.hazelcast.simulator.common.FailureType;
 import com.hazelcast.simulator.protocol.operation.FailureOperation;
 import com.hazelcast.simulator.protocol.registry.ComponentRegistry;
 import com.hazelcast.simulator.protocol.registry.TestData;
@@ -62,21 +63,24 @@ public class FailureCollector {
     public void notify(FailureOperation failure) {
         failure = enrich(failure);
 
-        WorkerData worker = componentRegistry.findWorker(failure.getWorkerAddress());
-        if (worker == null) {
-            // we are not interested in failures of workers that aren't registered any longer.
-            return;
-        }
+        if (failure.getType() != FailureType.WORKER_CREATE_ERROR) {
+            WorkerData worker = componentRegistry.findWorker(failure.getWorkerAddress());
+            if (worker == null) {
+                // we are not interested in failures of workers that aren't registered any longer.
+                return;
+            }
 
-        // it the failure is the terminal for that workers, we need to remove it from the component registry
-        if (failure.getType().isTerminal()) {
-            LOGGER.info("Removing worker " + worker.getAddress() + " from componentRegistry due to [" + failure.getType() + "]");
-            componentRegistry.removeWorker(worker.getAddress());
-        }
+            // it the failure is the terminal for that workers, we need to remove it from the component registry
+            if (failure.getType().isTerminal()) {
+                LOGGER.info("Removing worker " + worker.getAddress()
+                        + " from componentRegistry due to [" + failure.getType() + "]");
+                componentRegistry.removeWorker(worker.getAddress());
+            }
 
-        // if we don't care for the failure, we are done; no need to log anything.
-        if (worker.isIgnoreFailures() || failure.getType() == WORKER_NORMAL_EXIT) {
-            return;
+            // if we don't care for the failure, we are done; no need to log anything.
+            if (worker.isIgnoreFailures() || failure.getType() == WORKER_NORMAL_EXIT) {
+                return;
+            }
         }
 
         int failureCount = criticalFailureCounter.incrementAndGet();
@@ -107,6 +111,10 @@ public class FailureCollector {
     }
 
     private void logFailure(FailureOperation failure, long failureCount, boolean isCriticalFailure) {
+        if (failure.getType() == FailureType.WORKER_CREATE_ERROR) {
+            return;
+        }
+
         int failureNumber = failureNumberGenerator.incrementAndGet();
         if (failureCount < MAX_CONSOLE_FAILURE_COUNT) {
             if (isCriticalFailure) {
diff --git a/simulator/src/main/java/com/hazelcast/simulator/protocol/operation/FailureOperation.java b/simulator/src/main/java/com/hazelcast/simulator/protocol/operation/FailureOperation.java
index cc961b729..57f54a003 100644
--- a/simulator/src/main/java/com/hazelcast/simulator/protocol/operation/FailureOperation.java
+++ b/simulator/src/main/java/com/hazelcast/simulator/protocol/operation/FailureOperation.java
@@ -42,14 +42,14 @@ public class FailureOperation implements SimulatorOperation {
 
     public FailureOperation(String message, FailureType type, SimulatorAddress workerAddress, String agentAddress,
                             Throwable cause) {
-        this(message, type, workerAddress, agentAddress, null, null, null, throwableToString(cause));
+        this(message, type, workerAddress, agentAddress, null, null, null, cause == null ? "" : throwableToString(cause));
     }
 
     public FailureOperation(String message, FailureType type, SimulatorAddress workerAddress, String agentAddress,
                             String hzAddress, String workerId, String testId, String cause) {
         this.message = message;
         this.type = type.name();
-        this.workerAddress = (workerAddress == null) ? null : workerAddress.toString();
+        this.workerAddress = workerAddress == null ? null : workerAddress.toString();
         this.agentAddress = agentAddress;
         this.hzAddress = hzAddress;
         this.workerId = workerId;