Repository: hazelcast/hazelcast-simulator
Issue: Parallel test suites

Currently test suites can't be run in parallel, because in the Agent only 1 testsuite can be active.


Changed files:
simulator/src/main/java/com/hazelcast/simulator/protocol/processors/AgentOperationProcessor.java
simulator/src/main/java/com/hazelcast/simulator/agent/FailureHandlerImpl.java
simulator/src/test/java/com/hazelcast/simulator/coordinator/FailureCollectorTest.java
simulator/src/main/java/com/hazelcast/simulator/protocol/operation/OperationType.java
simulator/src/main/java/com/hazelcast/simulator/protocol/registry/TestData.java
simulator/src/test/java/com/hazelcast/simulator/protocol/processors/CoordinatorOperationProcessorTest.java
simulator/src/main/java/com/hazelcast/simulator/coordinator/FailureCollector.java
simulator/src/test/java/com/hazelcast/simulator/coordinator/RunTestSuiteTaskTest.java
simulator/src/test/java/com/hazelcast/simulator/protocol/processors/AgentOperationProcessorTest.java
simulator/src/test/java/com/hazelcast/simulator/protocol/ProtocolUtil.java
simulator/src/main/java/com/hazelcast/simulator/agent/Agent.java
simulator/src/main/java/com/hazelcast/simulator/coordinator/Coordinator.java
simulator/src/test/java/com/hazelcast/simulator/agent/FailureHandlerImplTest.java
simulator/src/test/java/com/hazelcast/simulator/agent/AgentSmokeTest.java
simulator/src/main/java/com/hazelcast/simulator/protocol/operation/FailureOperation.java
simulator/src/main/java/com/hazelcast/simulator/coordinator/RunTestSuiteTask.java
simulator/src/test/java/com/hazelcast/simulator/protocol/operation/FailureOperationTest.java
simulator/src/main/java/com/hazelcast/simulator/protocol/registry/ComponentRegistry.java
simulator/src/test/java/com/hazelcast/simulator/protocol/registry/ComponentRegistryTest.java
simulator/src/main/java/com/hazelcast/simulator/protocol/operation/InitTestSuiteOperation.java

diff --git a/simulator/src/main/java/com/hazelcast/simulator/agent/Agent.java b/simulator/src/main/java/com/hazelcast/simulator/agent/Agent.java
index 03c687d95..a87eec8d6 100644
--- a/simulator/src/main/java/com/hazelcast/simulator/agent/Agent.java
+++ b/simulator/src/main/java/com/hazelcast/simulator/agent/Agent.java
@@ -18,7 +18,6 @@ package com.hazelcast.simulator.agent;
 import com.hazelcast.simulator.agent.workerprocess.WorkerProcessFailureMonitor;
 import com.hazelcast.simulator.agent.workerprocess.WorkerProcessManager;
 import com.hazelcast.simulator.common.ShutdownThread;
-import com.hazelcast.simulator.common.TestSuite;
 import com.hazelcast.simulator.protocol.connector.AgentConnector;
 import com.hazelcast.simulator.protocol.operation.OperationTypeCounter;
 import org.apache.log4j.Logger;
@@ -43,7 +42,7 @@ public class Agent {
     private static final Logger LOGGER = Logger.getLogger(Agent.class);
     private static final AtomicBoolean SHUTDOWN_STARTED = new AtomicBoolean();
 
-     private final WorkerProcessManager workerProcessManager = new WorkerProcessManager();
+    private final WorkerProcessManager workerProcessManager = new WorkerProcessManager();
 
     private final int addressIndex;
     private final String publicAddress;
@@ -58,7 +57,6 @@ public class Agent {
     private final FailureHandlerImpl failureSender;
     private final WorkerProcessFailureMonitor workerProcessFailureMonitor;
 
-    private volatile TestSuite testSuite;
     private volatile String sessionId;
 
     public Agent(int addressIndex, String publicAddress, int port, String cloudProvider, String cloudIdentity,
@@ -86,7 +84,7 @@ public class Agent {
     }
 
     private void createPidFile() {
-         deleteQuiet(pidFile);
+        deleteQuiet(pidFile);
         writeText("" + getPID(), pidFile);
     }
 
@@ -118,15 +116,6 @@ public class Agent {
         this.sessionId = sessionId;
     }
 
-    public void setTestSuite(TestSuite testSuite) {
-        this.testSuite = testSuite;
-        failureSender.setTestSuite(testSuite);
-    }
-
-    public TestSuite getTestSuite() {
-        return testSuite;
-    }
-
     public File getSessionDirectory() {
         String sessionId = this.sessionId;
         if (sessionId == null) {
diff --git a/simulator/src/main/java/com/hazelcast/simulator/agent/FailureHandlerImpl.java b/simulator/src/main/java/com/hazelcast/simulator/agent/FailureHandlerImpl.java
index e9b341ab7..305a5b669 100644
--- a/simulator/src/main/java/com/hazelcast/simulator/agent/FailureHandlerImpl.java
+++ b/simulator/src/main/java/com/hazelcast/simulator/agent/FailureHandlerImpl.java
@@ -18,7 +18,6 @@ package com.hazelcast.simulator.agent;
 import com.hazelcast.simulator.agent.workerprocess.FailureHandler;
 import com.hazelcast.simulator.agent.workerprocess.WorkerProcess;
 import com.hazelcast.simulator.common.FailureType;
-import com.hazelcast.simulator.common.TestSuite;
 import com.hazelcast.simulator.protocol.connector.AgentConnector;
 import com.hazelcast.simulator.protocol.core.Response;
 import com.hazelcast.simulator.protocol.core.ResponseFuture;
@@ -38,8 +37,6 @@ class FailureHandlerImpl implements FailureHandler {
     private final String agentAddress;
     private final AgentConnector agentConnector;
 
-    private volatile TestSuite testSuite;
-
     private int failureCount;
 
     FailureHandlerImpl(String agentAddress, AgentConnector agentConnector) {
@@ -47,10 +44,6 @@ class FailureHandlerImpl implements FailureHandler {
         this.agentConnector = agentConnector;
     }
 
-    public void setTestSuite(TestSuite testSuite) {
-        this.testSuite = testSuite;
-    }
-
     @Override
     public boolean handle(String message, FailureType type, WorkerProcess workerProcess, String testId, String cause) {
         boolean send = true;
@@ -73,7 +66,7 @@ class FailureHandlerImpl implements FailureHandler {
         String workerId = workerProcess.getId();
 
         FailureOperation operation = new FailureOperation(message, type, workerAddress, agentAddress,
-                workerProcess.getHazelcastAddress(), workerId, testId, testSuite, cause);
+                workerProcess.getHazelcastAddress(), workerId, testId, cause);
 
         if (type.isPoisonPill()) {
             LOGGER.info(format("Worker %s (%s) finished.", workerId, workerAddress));
diff --git a/simulator/src/main/java/com/hazelcast/simulator/coordinator/Coordinator.java b/simulator/src/main/java/com/hazelcast/simulator/coordinator/Coordinator.java
index 75c40ca3a..11f8cf975 100644
--- a/simulator/src/main/java/com/hazelcast/simulator/coordinator/Coordinator.java
+++ b/simulator/src/main/java/com/hazelcast/simulator/coordinator/Coordinator.java
@@ -98,7 +98,7 @@ public final class Coordinator {
         this.outputDirectory = ensureNewDirectory(new File(getUserDir(), coordinatorParameters.getSessionId()));
         this.componentRegistry = componentRegistry;
         this.coordinatorParameters = coordinatorParameters;
-        this.failureCollector = new FailureCollector(outputDirectory);
+        this.failureCollector = new FailureCollector(outputDirectory, componentRegistry);
         this.failureCollector.addListener(true, new ComponentRegistryFailureListener(componentRegistry));
         this.simulatorProperties = coordinatorParameters.getSimulatorProperties();
         this.bash = new Bash(simulatorProperties);
diff --git a/simulator/src/main/java/com/hazelcast/simulator/coordinator/FailureCollector.java b/simulator/src/main/java/com/hazelcast/simulator/coordinator/FailureCollector.java
index 990daedba..472fbcfbc 100644
--- a/simulator/src/main/java/com/hazelcast/simulator/coordinator/FailureCollector.java
+++ b/simulator/src/main/java/com/hazelcast/simulator/coordinator/FailureCollector.java
@@ -17,6 +17,8 @@ package com.hazelcast.simulator.coordinator;
 
 import com.hazelcast.simulator.common.FailureType;
 import com.hazelcast.simulator.protocol.operation.FailureOperation;
+import com.hazelcast.simulator.protocol.registry.ComponentRegistry;
+import com.hazelcast.simulator.protocol.registry.TestData;
 import com.hazelcast.simulator.utils.CommandLineExitException;
 import org.apache.log4j.Logger;
 
@@ -47,9 +49,11 @@ public class FailureCollector {
     private final ConcurrentMap<String, Boolean> hasCriticalFailuresMap = new ConcurrentHashMap<String, Boolean>();
 
     private final File file;
+    private final ComponentRegistry componentRegistry;
 
-    public FailureCollector(File outputDirectory) {
+    public FailureCollector(File outputDirectory, ComponentRegistry componentRegistry) {
         this.file = new File(outputDirectory, "failures.txt");
+        this.componentRegistry = componentRegistry;
     }
 
     public void addListener(FailureListener listener) {
@@ -61,6 +65,8 @@ public class FailureCollector {
     }
 
     public void notify(FailureOperation failure) {
+        failure = enrichWithTestSuite(failure);
+
         boolean isFinishedFailure = false;
 
         FailureType failureType = failure.getType();
@@ -92,6 +98,21 @@ public class FailureCollector {
         }
     }
 
+    private FailureOperation enrichWithTestSuite(FailureOperation failure) {
+        String testId = failure.getTestId();
+        if (testId == null) {
+            return failure;
+        }
+
+        TestData testData = componentRegistry.getTest(testId);
+        if (testData == null) {
+            return failure;
+        }
+
+        failure.setTestSuite(testData.getTestSuite());
+        return failure;
+    }
+
     private void logFailure(FailureOperation failure, long failureCount, boolean isCriticalFailure) {
         int failureNumber = failureNumberGenerator.incrementAndGet();
         if (failureCount < MAX_CONSOLE_FAILURE_COUNT) {
diff --git a/simulator/src/main/java/com/hazelcast/simulator/coordinator/RunTestSuiteTask.java b/simulator/src/main/java/com/hazelcast/simulator/coordinator/RunTestSuiteTask.java
index d706c3d7b..f525396eb 100644
--- a/simulator/src/main/java/com/hazelcast/simulator/coordinator/RunTestSuiteTask.java
+++ b/simulator/src/main/java/com/hazelcast/simulator/coordinator/RunTestSuiteTask.java
@@ -19,7 +19,6 @@ package com.hazelcast.simulator.coordinator;
 import com.hazelcast.simulator.common.TestCase;
 import com.hazelcast.simulator.common.TestPhase;
 import com.hazelcast.simulator.common.TestSuite;
-import com.hazelcast.simulator.protocol.operation.InitTestSuiteOperation;
 import com.hazelcast.simulator.protocol.registry.ComponentRegistry;
 import com.hazelcast.simulator.protocol.registry.TargetType;
 import com.hazelcast.simulator.protocol.registry.TestData;
@@ -67,19 +66,17 @@ public class RunTestSuiteTask {
     }
 
     public void run() {
-        componentRegistry.addTests(testSuite);
+        List<TestData> tests = componentRegistry.addTests(testSuite);
         try {
-            run0();
+            run0(tests);
         } finally {
             testPhaseListeners.removeAllListeners(runners);
-            componentRegistry.removeTests();
+            componentRegistry.removeTests(testSuite);
             performanceStatsCollector.logDetailedPerformanceInfo(testSuite.getDurationSeconds());
         }
     }
 
-    private void run0() {
-        remoteClient.sendToAllAgents(new InitTestSuiteOperation(testSuite));
-
+    private void run0(List<TestData> tests) {
         int testCount = testSuite.size();
         boolean parallel = testSuite.isParallel() && testCount > 1;
         Map<TestPhase, CountDownLatch> testPhaseSyncMap = getTestPhaseSyncMap(testCount, parallel,
@@ -88,7 +85,7 @@ public class RunTestSuiteTask {
         echoer.echo("Starting TestSuite");
         echoTestSuiteDuration(parallel);
 
-        for (TestData testData : componentRegistry.getTests()) {
+        for (TestData testData: tests) {
             int testIndex = testData.getTestIndex();
             TestCase testCase = testData.getTestCase();
             echoer.echo("Configuration for %s (T%d):%n%s", testCase.getId(), testIndex, testCase);
diff --git a/simulator/src/main/java/com/hazelcast/simulator/protocol/operation/FailureOperation.java b/simulator/src/main/java/com/hazelcast/simulator/protocol/operation/FailureOperation.java
index adaf698e9..0aaf48a6c 100644
--- a/simulator/src/main/java/com/hazelcast/simulator/protocol/operation/FailureOperation.java
+++ b/simulator/src/main/java/com/hazelcast/simulator/protocol/operation/FailureOperation.java
@@ -37,16 +37,16 @@ public class FailureOperation implements SimulatorOperation {
     private final String hzAddress;
     private final String workerId;
     private final String testId;
-    private final TestSuite testSuite;
+    private TestSuite testSuite;
     private final String cause;
 
     public FailureOperation(String message, FailureType type, SimulatorAddress workerAddress, String agentAddress,
                             Throwable cause) {
-        this(message, type, workerAddress, agentAddress, null, null, null, null, throwableToString(cause));
+        this(message, type, workerAddress, agentAddress, null, null, null, throwableToString(cause));
     }
 
     public FailureOperation(String message, FailureType type, SimulatorAddress workerAddress, String agentAddress,
-                            String hzAddress, String workerId, String testId, TestSuite testSuite, String cause) {
+                            String hzAddress, String workerId, String testId, String cause) {
         this.message = message;
         this.type = type.name();
         this.workerAddress = (workerAddress == null) ? null : workerAddress.toString();
@@ -54,10 +54,14 @@ public class FailureOperation implements SimulatorOperation {
         this.hzAddress = hzAddress;
         this.workerId = workerId;
         this.testId = testId;
-        this.testSuite = testSuite;
         this.cause = cause;
     }
 
+    public FailureOperation setTestSuite(TestSuite testSuite) {
+        this.testSuite = testSuite;
+        return this;
+    }
+
     public FailureType getType() {
         return FailureType.valueOf(type);
     }
diff --git a/simulator/src/main/java/com/hazelcast/simulator/protocol/operation/InitTestSuiteOperation.java b/simulator/src/main/java/com/hazelcast/simulator/protocol/operation/InitTestSuiteOperation.java
deleted file mode 100644
index 956de1aa0..000000000
--- a/simulator/src/main/java/com/hazelcast/simulator/protocol/operation/InitTestSuiteOperation.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Copyright (c) 2008-2016, Hazelcast, Inc. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.hazelcast.simulator.protocol.operation;
-
-import com.hazelcast.simulator.common.TestSuite;
-
-public class InitTestSuiteOperation implements SimulatorOperation {
-
-    private final TestSuite testSuite;
-
-    public InitTestSuiteOperation(TestSuite testSuite) {
-        this.testSuite = testSuite;
-    }
-
-    public TestSuite getTestSuite() {
-        return testSuite;
-    }
-}
diff --git a/simulator/src/main/java/com/hazelcast/simulator/protocol/operation/OperationType.java b/simulator/src/main/java/com/hazelcast/simulator/protocol/operation/OperationType.java
index e475bbb44..057226a28 100644
--- a/simulator/src/main/java/com/hazelcast/simulator/protocol/operation/OperationType.java
+++ b/simulator/src/main/java/com/hazelcast/simulator/protocol/operation/OperationType.java
@@ -47,12 +47,11 @@ public enum OperationType {
 
     // AgentOperationProcessor
     INIT_SESSION(InitSessionOperation.class, 3000),
-    INIT_TEST_SUITE(InitTestSuiteOperation.class, 3001),
-    CREATE_WORKER(CreateWorkerOperation.class, 3002),
-    START_TIMEOUT_DETECTION(StartTimeoutDetectionOperation.class, 3003),
-    STOP_TIMEOUT_DETECTION(StopTimeoutDetectionOperation.class, 3004),
-    IGNORE_WORKER_FAILURE(IgnoreWorkerFailureOperation.class, 3005),
-    BASH(BashOperation.class, 3006),
+    CREATE_WORKER(CreateWorkerOperation.class, 3001),
+    START_TIMEOUT_DETECTION(StartTimeoutDetectionOperation.class, 3002),
+    STOP_TIMEOUT_DETECTION(StopTimeoutDetectionOperation.class, 3003),
+    IGNORE_WORKER_FAILURE(IgnoreWorkerFailureOperation.class, 3004),
+    BASH(BashOperation.class, 3005),
 
     // WorkerOperationProcessor
     PING(PingOperation.class, 4000),
diff --git a/simulator/src/main/java/com/hazelcast/simulator/protocol/processors/AgentOperationProcessor.java b/simulator/src/main/java/com/hazelcast/simulator/protocol/processors/AgentOperationProcessor.java
index a438d0179..1eea8cc6d 100644
--- a/simulator/src/main/java/com/hazelcast/simulator/protocol/processors/AgentOperationProcessor.java
+++ b/simulator/src/main/java/com/hazelcast/simulator/protocol/processors/AgentOperationProcessor.java
@@ -29,7 +29,6 @@ import com.hazelcast.simulator.protocol.operation.BashOperation;
 import com.hazelcast.simulator.protocol.operation.CreateWorkerOperation;
 import com.hazelcast.simulator.protocol.operation.IgnoreWorkerFailureOperation;
 import com.hazelcast.simulator.protocol.operation.InitSessionOperation;
-import com.hazelcast.simulator.protocol.operation.InitTestSuiteOperation;
 import com.hazelcast.simulator.protocol.operation.IntegrationTestOperation;
 import com.hazelcast.simulator.protocol.operation.LogOperation;
 import com.hazelcast.simulator.protocol.operation.OperationType;
@@ -83,9 +82,6 @@ public class AgentOperationProcessor extends AbstractOperationProcessor {
             case INIT_SESSION:
                 agent.setSessionId(((InitSessionOperation) operation).getSessionId());
                 break;
-            case INIT_TEST_SUITE:
-                agent.setTestSuite(((InitTestSuiteOperation) operation).getTestSuite());
-                break;
             case CREATE_WORKER:
                 return processCreateWorker((CreateWorkerOperation) operation);
             case START_TIMEOUT_DETECTION:
diff --git a/simulator/src/main/java/com/hazelcast/simulator/protocol/registry/ComponentRegistry.java b/simulator/src/main/java/com/hazelcast/simulator/protocol/registry/ComponentRegistry.java
index f198824a9..15223ea32 100644
--- a/simulator/src/main/java/com/hazelcast/simulator/protocol/registry/ComponentRegistry.java
+++ b/simulator/src/main/java/com/hazelcast/simulator/protocol/registry/ComponentRegistry.java
@@ -29,6 +29,7 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
@@ -243,8 +244,7 @@ public class ComponentRegistry {
                     agent.getAddress(),
                     formatLong(agentMemberWorkerCount, 2),
                     formatLong(agentClientWorkerCount, 2),
-                    agentVersionSpecs
-            ));
+                    agentVersionSpecs));
 
             for (WorkerData worker : agent.getWorkers()) {
                 LOGGER.info("        Worker " + worker.getAddress() + " " + worker.getSettings().getWorkerType()
@@ -268,17 +268,32 @@ public class ComponentRegistry {
         }
     }
 
-    public synchronized void addTests(TestSuite testSuite) {
+    public synchronized List<TestData> addTests(TestSuite testSuite) {
         for (TestCase testCase : testSuite.getTestCaseList()) {
-            int addressIndex = testIndex.incrementAndGet();
-            SimulatorAddress testAddress = new SimulatorAddress(AddressLevel.TEST, 0, 0, addressIndex);
-            TestData testData = new TestData(addressIndex, testAddress, testCase);
+            if (tests.containsKey(testCase.getId())) {
+                throw new IllegalArgumentException(format("Already a test running with id '%s'", testCase.getId()));
+            }
+        }
+
+        List<TestData> result = new ArrayList<TestData>(testSuite.size());
+        for (TestCase testCase : testSuite.getTestCaseList()) {
+            int testIndex = this.testIndex.incrementAndGet();
+            SimulatorAddress testAddress = new SimulatorAddress(AddressLevel.TEST, 0, 0, testIndex);
+            TestData testData = new TestData(testIndex, testAddress, testCase, testSuite);
+            result.add(testData);
             tests.put(testCase.getId(), testData);
         }
+        return result;
     }
 
-    public void removeTests() {
-        tests.clear();
+    public synchronized void removeTests(TestSuite testSuite) {
+        for (Map.Entry<String, TestData> entry : tests.entrySet()) {
+            TestData testData = entry.getValue();
+            if (testData.getTestSuite().equals(testSuite)) {
+                String testId = entry.getKey();
+                tests.remove(testId);
+            }
+        }
     }
 
     public int testCount() {
diff --git a/simulator/src/main/java/com/hazelcast/simulator/protocol/registry/TestData.java b/simulator/src/main/java/com/hazelcast/simulator/protocol/registry/TestData.java
index 06d6424e1..be9106f9f 100644
--- a/simulator/src/main/java/com/hazelcast/simulator/protocol/registry/TestData.java
+++ b/simulator/src/main/java/com/hazelcast/simulator/protocol/registry/TestData.java
@@ -16,6 +16,7 @@
 package com.hazelcast.simulator.protocol.registry;
 
 import com.hazelcast.simulator.common.TestCase;
+import com.hazelcast.simulator.common.TestSuite;
 import com.hazelcast.simulator.protocol.core.SimulatorAddress;
 
 public class TestData {
@@ -23,11 +24,13 @@ public class TestData {
     private final int testIndex;
     private final SimulatorAddress address;
     private final TestCase testCase;
+    private final TestSuite testSuite;
 
-    TestData(int testIndex, SimulatorAddress address, TestCase testCase) {
+    TestData(int testIndex, SimulatorAddress address, TestCase testCase, TestSuite testSuite) {
         this.testIndex = testIndex;
         this.address = address;
         this.testCase = testCase;
+        this.testSuite = testSuite;
     }
 
     public int getTestIndex() {
@@ -41,4 +44,8 @@ public class TestData {
     public TestCase getTestCase() {
         return testCase;
     }
+
+    public TestSuite getTestSuite() {
+        return testSuite;
+    }
 }
diff --git a/simulator/src/test/java/com/hazelcast/simulator/agent/AgentSmokeTest.java b/simulator/src/test/java/com/hazelcast/simulator/agent/AgentSmokeTest.java
index ec3bf6bf4..ec885ae6c 100644
--- a/simulator/src/test/java/com/hazelcast/simulator/agent/AgentSmokeTest.java
+++ b/simulator/src/test/java/com/hazelcast/simulator/agent/AgentSmokeTest.java
@@ -21,7 +21,6 @@ import com.hazelcast.simulator.protocol.core.SimulatorAddress;
 import com.hazelcast.simulator.protocol.operation.CreateTestOperation;
 import com.hazelcast.simulator.protocol.operation.FailureOperation;
 import com.hazelcast.simulator.protocol.operation.InitSessionOperation;
-import com.hazelcast.simulator.protocol.operation.InitTestSuiteOperation;
 import com.hazelcast.simulator.protocol.operation.StartTestOperation;
 import com.hazelcast.simulator.protocol.operation.StartTestPhaseOperation;
 import com.hazelcast.simulator.protocol.operation.StopTestOperation;
@@ -101,7 +100,7 @@ public class AgentSmokeTest implements FailureListener {
         testPhaseListeners = new TestPhaseListeners();
         PerformanceStatsCollector performanceStatsCollector = new PerformanceStatsCollector();
         outputDirectory = createTmpDirectory();
-        failureCollector = new FailureCollector(outputDirectory);
+        failureCollector = new FailureCollector(outputDirectory, componentRegistry);
 
         CoordinatorOperationProcessor processor = new CoordinatorOperationProcessor(
                 null, failureCollector, testPhaseListeners, performanceStatsCollector);
@@ -186,7 +185,6 @@ public class AgentSmokeTest implements FailureListener {
         try {
             String testId = testCase.getId();
             TestSuite testSuite = new TestSuite();
-            remoteClient.sendToAllAgents(new InitTestSuiteOperation(testSuite));
             testSuite.addTest(testCase);
 
             componentRegistry.addTests(testSuite);
@@ -224,7 +222,7 @@ public class AgentSmokeTest implements FailureListener {
             runPhase(testPhaseListener, testCase, TestPhase.GLOBAL_TEARDOWN);
             runPhase(testPhaseListener, testCase, TestPhase.LOCAL_TEARDOWN);
         } finally {
-            componentRegistry.removeTests();
+           // componentRegistry.removeTests();
 
             LOGGER.info("Terminating workers...");
             new TerminateWorkersTask(simulatorProperties, componentRegistry, remoteClient).run();
diff --git a/simulator/src/test/java/com/hazelcast/simulator/agent/FailureHandlerImplTest.java b/simulator/src/test/java/com/hazelcast/simulator/agent/FailureHandlerImplTest.java
index d16d749d5..85332dd7c 100644
--- a/simulator/src/test/java/com/hazelcast/simulator/agent/FailureHandlerImplTest.java
+++ b/simulator/src/test/java/com/hazelcast/simulator/agent/FailureHandlerImplTest.java
@@ -62,10 +62,7 @@ public class FailureHandlerImplTest {
         when(agentConnector.write(any(SimulatorAddress.class), any(SimulatorOperation.class))).thenReturn(response);
         when(agentConnector.getFutureMap()).thenReturn(futureMap);
 
-        TestSuite testSuite = new TestSuite();
-
         failureSender = new FailureHandlerImpl("127.0.0.1", agentConnector);
-        failureSender.setTestSuite(testSuite);
     }
 
     @Test
diff --git a/simulator/src/test/java/com/hazelcast/simulator/coordinator/FailureCollectorTest.java b/simulator/src/test/java/com/hazelcast/simulator/coordinator/FailureCollectorTest.java
index 7eddd88d1..26ed1a649 100644
--- a/simulator/src/test/java/com/hazelcast/simulator/coordinator/FailureCollectorTest.java
+++ b/simulator/src/test/java/com/hazelcast/simulator/coordinator/FailureCollectorTest.java
@@ -3,6 +3,7 @@ package com.hazelcast.simulator.coordinator;
 import com.hazelcast.simulator.protocol.core.AddressLevel;
 import com.hazelcast.simulator.protocol.core.SimulatorAddress;
 import com.hazelcast.simulator.protocol.operation.FailureOperation;
+import com.hazelcast.simulator.protocol.registry.ComponentRegistry;
 import com.hazelcast.simulator.utils.CommandLineExitException;
 import com.hazelcast.simulator.utils.TestUtils;
 import org.junit.After;
@@ -26,23 +27,25 @@ public class FailureCollectorTest {
     private FailureOperation oomOperation;
     private FailureOperation finishedOperation;
     private File outputDirectory;
+    private ComponentRegistry componentRegistry;
 
     @Before
     public void setUp() {
         outputDirectory = TestUtils.createTmpDirectory();
-        failureCollector = new FailureCollector(outputDirectory);
+        componentRegistry = new ComponentRegistry();
+        failureCollector = new FailureCollector(outputDirectory, componentRegistry);
 
         SimulatorAddress workerAddress = new SimulatorAddress(AddressLevel.WORKER, 1, 1, 0);
         String agentAddress = workerAddress.getParent().toString();
 
         exceptionOperation = new FailureOperation("exception", WORKER_EXCEPTION, workerAddress, agentAddress,
-                "127.0.0.1:5701", "workerId", "testId", null, null);
+                "127.0.0.1:5701", "workerId", "testId", null);
 
         oomOperation = new FailureOperation("oom", WORKER_OOM, workerAddress, agentAddress,
-                "127.0.0.1:5701", "workerId", "testId", null, null);
+                "127.0.0.1:5701", "workerId", "testId", null);
 
         finishedOperation = new FailureOperation("finished", WORKER_FINISHED, workerAddress, agentAddress,
-                "127.0.0.1:5701", "workerId", "testId", null, null);
+                "127.0.0.1:5701", "workerId", "testId", null);
     }
 
     @After
diff --git a/simulator/src/test/java/com/hazelcast/simulator/coordinator/RunTestSuiteTaskTest.java b/simulator/src/test/java/com/hazelcast/simulator/coordinator/RunTestSuiteTaskTest.java
index d20181033..3c0412b46 100644
--- a/simulator/src/test/java/com/hazelcast/simulator/coordinator/RunTestSuiteTaskTest.java
+++ b/simulator/src/test/java/com/hazelcast/simulator/coordinator/RunTestSuiteTaskTest.java
@@ -5,6 +5,7 @@ import com.hazelcast.simulator.common.SimulatorProperties;
 import com.hazelcast.simulator.common.TestCase;
 import com.hazelcast.simulator.common.TestPhase;
 import com.hazelcast.simulator.common.TestSuite;
+import com.hazelcast.simulator.common.WorkerType;
 import com.hazelcast.simulator.protocol.connector.CoordinatorConnector;
 import com.hazelcast.simulator.protocol.core.Response;
 import com.hazelcast.simulator.protocol.core.ResponseType;
@@ -18,7 +19,6 @@ import com.hazelcast.simulator.protocol.operation.StopTestOperation;
 import com.hazelcast.simulator.protocol.registry.ComponentRegistry;
 import com.hazelcast.simulator.protocol.registry.TargetType;
 import com.hazelcast.simulator.protocol.registry.TestData;
-import com.hazelcast.simulator.common.WorkerType;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
@@ -31,6 +31,7 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicLong;
 
 import static com.hazelcast.simulator.TestEnvironmentUtils.setupFakeEnvironment;
 import static com.hazelcast.simulator.TestEnvironmentUtils.tearDownFakeEnvironment;
@@ -62,6 +63,7 @@ import static org.mockito.Mockito.when;
 
 public class RunTestSuiteTaskTest {
 
+    private static final AtomicLong ID_GENERATOR = new AtomicLong();
     private CountDownLatch finishWorkerLatch = new CountDownLatch(1);
 
     private File outputDirectory;
@@ -90,14 +92,14 @@ public class RunTestSuiteTaskTest {
     @Before
     public void setUp() {
         testSuite = new TestSuite();
-        testSuite.addTest(new TestCase("CoordinatorTest1"));
-        testSuite.addTest(new TestCase("CoordinatorTest2"));
+        testSuite.addTest(new TestCase("CoordinatorTest" + ID_GENERATOR.incrementAndGet()));
+        testSuite.addTest(new TestCase("CoordinatorTest" + ID_GENERATOR.incrementAndGet()));
 
         outputDirectory = createTmpDirectory();
 
         SimulatorAddress address = new SimulatorAddress(WORKER, 1, 1, 0);
         criticalFailureOperation = new FailureOperation("expected critical failure", WORKER_EXCEPTION, address, "127.0.0.1",
-                "127.0.0.1:5701", "workerId", "CoordinatorTest1", testSuite, "stacktrace");
+                "127.0.0.1:5701", "workerId", "CoordinatorTest1", "stacktrace");
 
         simulatorProperties = new SimulatorProperties();
 
@@ -194,7 +196,7 @@ public class RunTestSuiteTaskTest {
 
     @Test
     public void runSequential_withSingleTest() {
-        TestCase testCase = new TestCase("CoordinatorTest");
+        TestCase testCase = new TestCase("CoordinatorTest" + ID_GENERATOR.incrementAndGet());
 
         testSuite = new TestSuite();
         testSuite.addTest(testCase);
@@ -208,7 +210,7 @@ public class RunTestSuiteTaskTest {
 
     @Test
     public void runParallel_withSingleTest() {
-        TestCase testCase = new TestCase("CoordinatorTest");
+        TestCase testCase = new TestCase("CoordinatorTest" + ID_GENERATOR.incrementAndGet());
 
         testSuite = new TestSuite();
         testSuite.addTest(testCase);
@@ -308,9 +310,8 @@ public class RunTestSuiteTaskTest {
         componentRegistry = new ComponentRegistry();
         componentRegistry.addAgent("127.0.0.1", "127.0.0.1");
         componentRegistry.addWorkers(componentRegistry.getFirstAgent().getAddress(), singletonList(workerProcessSettings));
-        componentRegistry.addTests(testSuite);
 
-        failureCollector = new FailureCollector(outputDirectory);
+        failureCollector = new FailureCollector(outputDirectory, componentRegistry);
         PerformanceStatsCollector performanceStatsCollector = new PerformanceStatsCollector();
         TestPhaseListeners testPhaseListeners = new TestPhaseListeners();
 
@@ -438,7 +439,7 @@ public class RunTestSuiteTaskTest {
             if (finishWorkerLatch != null) {
                 await(finishWorkerLatch);
                 FailureOperation operation = new FailureOperation("Worker finished", WORKER_FINISHED, workerAddress, "127.0.0.1",
-                        "127.0.0.1:5701", "workerId", "testId", testSuite, "stacktrace");
+                        "127.0.0.1:5701", "workerId", "testId", "stacktrace");
                 failureCollector.notify(operation);
             }
         }
diff --git a/simulator/src/test/java/com/hazelcast/simulator/protocol/ProtocolUtil.java b/simulator/src/test/java/com/hazelcast/simulator/protocol/ProtocolUtil.java
index 0c91c3cb1..400adbca2 100644
--- a/simulator/src/test/java/com/hazelcast/simulator/protocol/ProtocolUtil.java
+++ b/simulator/src/test/java/com/hazelcast/simulator/protocol/ProtocolUtil.java
@@ -19,6 +19,7 @@ import com.hazelcast.simulator.protocol.operation.IntegrationTestOperation;
 import com.hazelcast.simulator.protocol.operation.SimulatorOperation;
 import com.hazelcast.simulator.protocol.processors.CoordinatorOperationProcessor;
 import com.hazelcast.simulator.protocol.processors.TestOperationProcessor;
+import com.hazelcast.simulator.protocol.registry.ComponentRegistry;
 import com.hazelcast.simulator.utils.TestUtils;
 import com.hazelcast.simulator.utils.ThreadSpawner;
 import com.hazelcast.simulator.worker.Worker;
@@ -163,7 +164,8 @@ class ProtocolUtil {
         TestPhaseListeners testPhaseListeners = new TestPhaseListeners();
         PerformanceStatsCollector performanceStatsCollector = new PerformanceStatsCollector();
         File outputDirectory = TestUtils.createTmpDirectory();
-        FailureCollector failureCollector = new FailureCollector(outputDirectory);
+        ComponentRegistry componentRegistry = new ComponentRegistry();
+        FailureCollector failureCollector = new FailureCollector(outputDirectory, componentRegistry);
         CoordinatorOperationProcessor operationProcessor = new CoordinatorOperationProcessor(
                 null, failureCollector,testPhaseListeners,performanceStatsCollector);
         CoordinatorConnector coordinatorConnector = new CoordinatorConnector(operationProcessor, coordinatorPort);
diff --git a/simulator/src/test/java/com/hazelcast/simulator/protocol/operation/FailureOperationTest.java b/simulator/src/test/java/com/hazelcast/simulator/protocol/operation/FailureOperationTest.java
index 03be958fc..49d9d76a2 100644
--- a/simulator/src/test/java/com/hazelcast/simulator/protocol/operation/FailureOperationTest.java
+++ b/simulator/src/test/java/com/hazelcast/simulator/protocol/operation/FailureOperationTest.java
@@ -35,7 +35,7 @@ public class FailureOperationTest {
         cause = new TestException("expected exception");
         operation = new FailureOperation("FailureOperationTest", WORKER_EXCEPTION, workerAddress, null, cause);
         fullOperation = new FailureOperation("FailureOperationTest", WORKER_EXCEPTION, workerAddress, null, "127.0.0.1:5701",
-                "C_A1_W1-member", TEST_ID, testSuite, null);
+                "C_A1_W1-member", TEST_ID, null).setTestSuite(testSuite);
     }
 
     @Test
diff --git a/simulator/src/test/java/com/hazelcast/simulator/protocol/processors/AgentOperationProcessorTest.java b/simulator/src/test/java/com/hazelcast/simulator/protocol/processors/AgentOperationProcessorTest.java
index 353d8c0b2..b65815611 100644
--- a/simulator/src/test/java/com/hazelcast/simulator/protocol/processors/AgentOperationProcessorTest.java
+++ b/simulator/src/test/java/com/hazelcast/simulator/protocol/processors/AgentOperationProcessorTest.java
@@ -11,7 +11,6 @@ import com.hazelcast.simulator.protocol.connector.AgentConnector;
 import com.hazelcast.simulator.protocol.core.ResponseType;
 import com.hazelcast.simulator.protocol.operation.CreateTestOperation;
 import com.hazelcast.simulator.protocol.operation.CreateWorkerOperation;
-import com.hazelcast.simulator.protocol.operation.InitTestSuiteOperation;
 import com.hazelcast.simulator.protocol.operation.IntegrationTestOperation;
 import com.hazelcast.simulator.protocol.operation.SimulatorOperation;
 import com.hazelcast.simulator.protocol.operation.StartTimeoutDetectionOperation;
@@ -79,7 +78,6 @@ public class AgentOperationProcessorTest {
         agent = mock(Agent.class);
         when(agent.getAddressIndex()).thenReturn(1);
         when(agent.getPublicAddress()).thenReturn("127.0.0.1");
-        when(agent.getTestSuite()).thenReturn(testSuite);
         when(agent.getSessionId()).thenReturn("AgentOperationProcessorTest");
         when(agent.getSessionDirectory()).thenReturn(sessionDir);
         when(agent.getAgentConnector()).thenReturn(agentConnector);
@@ -112,16 +110,6 @@ public class AgentOperationProcessorTest {
         assertEquals(UNSUPPORTED_OPERATION_ON_THIS_PROCESSOR, responseType);
     }
 
-    @Test
-    public void testInitTestSuiteOperation() throws Exception {
-        SimulatorOperation operation = new InitTestSuiteOperation(testSuite);
-        ResponseType responseType = processor.processOperation(getOperationType(operation), operation, COORDINATOR);
-
-        assertEquals(SUCCESS, responseType);
-
-        verify(agent).setTestSuite(any(TestSuite.class));
-    }
-
     @Test(timeout = DEFAULT_TEST_TIMEOUT)
     public void testCreateWorkerOperation() throws Exception {
         ResponseType responseType = testCreateWorkerOperation(false, DEFAULT_STARTUP_TIMEOUT);
diff --git a/simulator/src/test/java/com/hazelcast/simulator/protocol/processors/CoordinatorOperationProcessorTest.java b/simulator/src/test/java/com/hazelcast/simulator/protocol/processors/CoordinatorOperationProcessorTest.java
index d80f3616e..2321fb208 100644
--- a/simulator/src/test/java/com/hazelcast/simulator/protocol/processors/CoordinatorOperationProcessorTest.java
+++ b/simulator/src/test/java/com/hazelcast/simulator/protocol/processors/CoordinatorOperationProcessorTest.java
@@ -14,6 +14,7 @@ import com.hazelcast.simulator.protocol.operation.IntegrationTestOperation;
 import com.hazelcast.simulator.protocol.operation.PerformanceStatsOperation;
 import com.hazelcast.simulator.protocol.operation.PhaseCompletedOperation;
 import com.hazelcast.simulator.protocol.operation.SimulatorOperation;
+import com.hazelcast.simulator.protocol.registry.ComponentRegistry;
 import com.hazelcast.simulator.test.TestException;
 import com.hazelcast.simulator.utils.TestUtils;
 import com.hazelcast.simulator.worker.performance.PerformanceStats;
@@ -54,6 +55,7 @@ public class CoordinatorOperationProcessorTest implements FailureListener {
 
     private CoordinatorOperationProcessor processor;
     private File outputDirectory;
+    private ComponentRegistry componentRegistry;
 
     @Before
     public void setUp() {
@@ -62,8 +64,9 @@ public class CoordinatorOperationProcessorTest implements FailureListener {
         testPhaseListeners = new TestPhaseListeners();
         performanceStatsCollector = new PerformanceStatsCollector();
 
+        componentRegistry = new ComponentRegistry();
         outputDirectory = TestUtils.createTmpDirectory();
-        failureCollector = new FailureCollector(outputDirectory);
+        failureCollector = new FailureCollector(outputDirectory, componentRegistry);
 
         processor = new CoordinatorOperationProcessor(null, failureCollector, testPhaseListeners,
                 performanceStatsCollector);
@@ -82,17 +85,6 @@ public class CoordinatorOperationProcessorTest implements FailureListener {
         assertEquals(UNSUPPORTED_OPERATION_ON_THIS_PROCESSOR, responseType);
     }
 
-//    @Test
-//    public void processException() {
-//        TestException exception = new TestException("expected exception");
-//        ExceptionOperation operation = new ExceptionOperation(WORKER_EXCEPTION.name(), "C_A1_W1", "FailingTest", exception);
-//
-//        ResponseType responseType = processor.process(operation, workerAddress);
-//
-//        assertEquals(SUCCESS, responseType);
-//        //assertEquals(1, exceptionLogger.getExceptionCount());
-//    }
-
     @Test
     public void processFailureOperation() {
         failureCollector.addListener(this);
diff --git a/simulator/src/test/java/com/hazelcast/simulator/protocol/registry/ComponentRegistryTest.java b/simulator/src/test/java/com/hazelcast/simulator/protocol/registry/ComponentRegistryTest.java
index 9f78641eb..e1ce4b26b 100644
--- a/simulator/src/test/java/com/hazelcast/simulator/protocol/registry/ComponentRegistryTest.java
+++ b/simulator/src/test/java/com/hazelcast/simulator/protocol/registry/ComponentRegistryTest.java
@@ -8,6 +8,7 @@ import com.hazelcast.simulator.protocol.core.SimulatorAddress;
 import com.hazelcast.simulator.protocol.registry.AgentData.AgentWorkerMode;
 import com.hazelcast.simulator.utils.CommandLineExitException;
 import com.hazelcast.simulator.common.WorkerType;
+import junit.framework.TestResult;
 import org.junit.Test;
 
 import java.util.ArrayList;
@@ -16,6 +17,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import static junit.framework.TestCase.assertSame;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
@@ -280,20 +282,30 @@ public class ComponentRegistryTest {
         testSuite.addTest(new TestCase("Test2"));
         testSuite.addTest(new TestCase("Test3"));
 
-        componentRegistry.addTests(testSuite);
+        List<TestData> tests = componentRegistry.addTests(testSuite);
 
+        assertEquals(3, tests.size());
         assertEquals(3, componentRegistry.testCount());
     }
 
     @Test
     public void testRemoveTests() {
-        TestSuite testSuite = new TestSuite();
-        testSuite.addTest(new TestCase("Test1"));
-        componentRegistry.addTests(testSuite);
+        TestSuite testSuite1 = new TestSuite()
+                .addTest(new TestCase("Test1a"))
+                .addTest(new TestCase("Test1b"));
+        componentRegistry.addTests(testSuite1);
+
+        TestSuite testSuite2 = new TestSuite()
+                .addTest(new TestCase("Test2a"))
+                .addTest(new TestCase("Test2b"));
+        componentRegistry.addTests(testSuite2);
 
-        componentRegistry.removeTests();
+        componentRegistry.removeTests(testSuite1);
 
-        assertEquals(0, componentRegistry.testCount());
+        assertEquals(2, componentRegistry.testCount());
+        for(TestData testData: componentRegistry.getTests()){
+            assertSame(testSuite2, testData.getTestSuite());
+        }
     }
 
     @Test