Repository: EnMasseProject/enmasse
Pull request #2116 (2018-12-12 18:31:07)

New marathon test with restarting pods


Patch:
diff --git a/Jenkinsfile-long b/Jenkinsfile-long
index 2dc3435a4..9e9f5b478 100644
--- a/Jenkinsfile-long
+++ b/Jenkinsfile-long
@@ -74,7 +74,9 @@ pipeline {
             }
         }
         failure {
-            echo "build failed"
+            script {
+                lib.sendMail(env.MAILING_LIST, env.JOB_NAME, env.BUILD_URL)
+            }
         }
     }
 }
diff --git a/agent/npm-shrinkwrap.json b/agent/npm-shrinkwrap.json
index 505dd09ab..7e2da30a1 100644
--- a/agent/npm-shrinkwrap.json
+++ b/agent/npm-shrinkwrap.json
@@ -366,6 +366,16 @@
                 "text-table": "0.2.0"
             },
             "dependencies": {
+                "acorn": {
+                    "version": "6.0.4",
+                    "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.0.4.tgz",
+                    "integrity": "sha512-VY4i5EKSKkofY2I+6QLTbTTN/UvEQPCo6eiwzzSaSWfpaDhOmStMCMod6wmuPciNq+XS0faCglFu2lHZpdHUtg=="
+                },
+                "acorn-jsx": {
+                    "version": "5.0.1",
+                    "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.0.1.tgz",
+                    "integrity": "sha512-HJ7CfNHrfJLlNTzIEUTj43LNWGkqpRLxm3YjAlcD0ACydk9XynzYsCBHxut+iqt+1aBXkx9UP/w/ZqMr13XIzg=="
+                },
                 "debug": {
                     "version": "4.1.0",
                     "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.0.tgz",
@@ -373,6 +383,16 @@
                     "requires": {
                         "ms": "2.1.1"
                     }
+                },
+                "espree": {
+                    "version": "5.0.0",
+                    "resolved": "https://registry.npmjs.org/espree/-/espree-5.0.0.tgz",
+                    "integrity": "sha512-1MpUfwsdS9MMoN7ZXqAr9e9UKdVHDcvrJpyx7mm1WuQlx/ygErEQBzgi5Nh5qBHIoYweprhtMkTCb9GhcAIcsA==",
+                    "requires": {
+                        "acorn": "6.0.4",
+                        "acorn-jsx": "5.0.1",
+                        "eslint-visitor-keys": "1.0.0"
+                    }
                 }
             }
         },
diff --git a/api-model/src/main/java/io/enmasse/admin/model/v1/BrokeredInfraConfigSpec.java b/api-model/src/main/java/io/enmasse/admin/model/v1/BrokeredInfraConfigSpec.java
index cb527812d..93c8560c6 100644
--- a/api-model/src/main/java/io/enmasse/admin/model/v1/BrokeredInfraConfigSpec.java
+++ b/api-model/src/main/java/io/enmasse/admin/model/v1/BrokeredInfraConfigSpec.java
@@ -43,6 +43,16 @@ public class BrokeredInfraConfigSpec extends AbstractWithAdditionalProperties {
                 Objects.equals(broker, that.broker);
     }
 
+    @Override
+    public String toString() {
+        return "BrokeredInfraConfigSpec{" +
+                "version='" + version + '\'' +
+                ", networkPolicy=" + networkPolicy +
+                ", admin=" + admin +
+                ", broker=" + broker +
+                '}';
+    }
+
     @Override
     public int hashCode() {
         return Objects.hash(version, networkPolicy, admin, broker);
diff --git a/api-model/src/main/java/io/enmasse/admin/model/v1/BrokeredInfraConfigSpecAdmin.java b/api-model/src/main/java/io/enmasse/admin/model/v1/BrokeredInfraConfigSpecAdmin.java
index 5078a9232..490b07482 100644
--- a/api-model/src/main/java/io/enmasse/admin/model/v1/BrokeredInfraConfigSpecAdmin.java
+++ b/api-model/src/main/java/io/enmasse/admin/model/v1/BrokeredInfraConfigSpecAdmin.java
@@ -39,6 +39,13 @@ public class BrokeredInfraConfigSpecAdmin extends AbstractWithAdditionalProperti
         return Objects.equals(resources, that.resources);
     }
 
+    @Override
+    public String toString() {
+        return "BrokeredInfraConfigSpecAdmin{" +
+                "resources=" + resources +
+                '}';
+    }
+
     @Override
     public int hashCode() {
         return Objects.hash(resources);
diff --git a/api-model/src/main/java/io/enmasse/admin/model/v1/BrokeredInfraConfigSpecAdminResources.java b/api-model/src/main/java/io/enmasse/admin/model/v1/BrokeredInfraConfigSpecAdminResources.java
index 00485004f..95587d9a4 100644
--- a/api-model/src/main/java/io/enmasse/admin/model/v1/BrokeredInfraConfigSpecAdminResources.java
+++ b/api-model/src/main/java/io/enmasse/admin/model/v1/BrokeredInfraConfigSpecAdminResources.java
@@ -39,6 +39,13 @@ public class BrokeredInfraConfigSpecAdminResources extends AbstractWithAdditiona
         return Objects.equals(memory, that.memory);
     }
 
+    @Override
+    public String toString() {
+        return "BrokeredInfraConfigSpecAdminResources{" +
+                "memory='" + memory + '\'' +
+                '}';
+    }
+
     @Override
     public int hashCode() {
         return Objects.hash(memory);
diff --git a/api-model/src/main/java/io/enmasse/admin/model/v1/BrokeredInfraConfigSpecBroker.java b/api-model/src/main/java/io/enmasse/admin/model/v1/BrokeredInfraConfigSpecBroker.java
index 0b4b64800..af55c4e90 100644
--- a/api-model/src/main/java/io/enmasse/admin/model/v1/BrokeredInfraConfigSpecBroker.java
+++ b/api-model/src/main/java/io/enmasse/admin/model/v1/BrokeredInfraConfigSpecBroker.java
@@ -31,6 +31,7 @@ public class BrokeredInfraConfigSpecBroker extends AbstractWithAdditionalPropert
     }
 
     public BrokeredInfraConfigSpecBroker(final BrokeredInfraConfigSpecBrokerResources resources) {
+        setResources(resources);
     }
 
     @Override
@@ -49,6 +50,16 @@ public class BrokeredInfraConfigSpecBroker extends AbstractWithAdditionalPropert
         return Objects.hash(resources, addressFullPolicy, storageClassName, updatePersistentVolumeClaim);
     }
 
+    @Override
+    public String toString() {
+        return "BrokeredInfraConfigSpecBroker{" +
+                "resources=" + resources +
+                ", addressFullPolicy='" + addressFullPolicy + '\'' +
+                ", storageClassName='" + storageClassName + '\'' +
+                ", updatePersistentVolumeClaim=" + updatePersistentVolumeClaim +
+                '}';
+    }
+
     public void setResources(BrokeredInfraConfigSpecBrokerResources resources) {
         this.resources = resources;
     }
diff --git a/api-model/src/main/java/io/enmasse/admin/model/v1/BrokeredInfraConfigSpecBrokerResources.java b/api-model/src/main/java/io/enmasse/admin/model/v1/BrokeredInfraConfigSpecBrokerResources.java
index fc004511f..4dd82a4cb 100644
--- a/api-model/src/main/java/io/enmasse/admin/model/v1/BrokeredInfraConfigSpecBrokerResources.java
+++ b/api-model/src/main/java/io/enmasse/admin/model/v1/BrokeredInfraConfigSpecBrokerResources.java
@@ -42,6 +42,14 @@ public class BrokeredInfraConfigSpecBrokerResources extends AbstractWithAddition
                 Objects.equals(storage, that.storage);
     }
 
+    @Override
+    public String toString() {
+        return "BrokeredInfraConfigSpecBrokerResources{" +
+                "memory='" + memory + '\'' +
+                ", storage='" + storage + '\'' +
+                '}';
+    }
+
     @Override
     public int hashCode() {
         return Objects.hash(memory, storage);
diff --git a/api-model/src/main/java/io/enmasse/admin/model/v1/StandardInfraConfigSpec.java b/api-model/src/main/java/io/enmasse/admin/model/v1/StandardInfraConfigSpec.java
index a0284ada9..21bc0ab12 100644
--- a/api-model/src/main/java/io/enmasse/admin/model/v1/StandardInfraConfigSpec.java
+++ b/api-model/src/main/java/io/enmasse/admin/model/v1/StandardInfraConfigSpec.java
@@ -78,6 +78,17 @@ public class StandardInfraConfigSpec extends AbstractWithAdditionalProperties {
         return Objects.hash(version, networkPolicy, admin, broker, router);
     }
 
+    @Override
+    public String toString() {
+        return "StandardInfraConfigSpec{" +
+                "version='" + version + '\'' +
+                ", networkPolicy=" + networkPolicy +
+                ", admin=" + admin +
+                ", broker=" + broker +
+                ", router=" + router +
+                '}';
+    }
+
     public void setNetworkPolicy(NetworkPolicy networkPolicy) {
         this.networkPolicy = networkPolicy;
     }
diff --git a/api-model/src/main/java/io/enmasse/admin/model/v1/StandardInfraConfigSpecAdmin.java b/api-model/src/main/java/io/enmasse/admin/model/v1/StandardInfraConfigSpecAdmin.java
index f5c5e6782..87e357562 100644
--- a/api-model/src/main/java/io/enmasse/admin/model/v1/StandardInfraConfigSpecAdmin.java
+++ b/api-model/src/main/java/io/enmasse/admin/model/v1/StandardInfraConfigSpecAdmin.java
@@ -53,4 +53,10 @@ public class StandardInfraConfigSpecAdmin extends AbstractWithAdditionalProperti
         return Objects.hash(resources);
     }
 
+    @Override
+    public String toString() {
+        return "StandardInfraConfigSpecAdmin{" +
+                "resources=" + resources +
+                '}';
+    }
 }
diff --git a/api-model/src/main/java/io/enmasse/admin/model/v1/StandardInfraConfigSpecAdminResources.java b/api-model/src/main/java/io/enmasse/admin/model/v1/StandardInfraConfigSpecAdminResources.java
index 35702bac5..906eb00ff 100644
--- a/api-model/src/main/java/io/enmasse/admin/model/v1/StandardInfraConfigSpecAdminResources.java
+++ b/api-model/src/main/java/io/enmasse/admin/model/v1/StandardInfraConfigSpecAdminResources.java
@@ -52,4 +52,10 @@ public class StandardInfraConfigSpecAdminResources extends AbstractWithAdditiona
         return Objects.hash(memory);
     }
 
+    @Override
+    public String toString() {
+        return "StandardInfraConfigSpecAdminResources{" +
+                "memory='" + memory + '\'' +
+                '}';
+    }
 }
diff --git a/api-model/src/main/java/io/enmasse/admin/model/v1/StandardInfraConfigSpecBroker.java b/api-model/src/main/java/io/enmasse/admin/model/v1/StandardInfraConfigSpecBroker.java
index 38fb7aa0b..efcc3d21a 100644
--- a/api-model/src/main/java/io/enmasse/admin/model/v1/StandardInfraConfigSpecBroker.java
+++ b/api-model/src/main/java/io/enmasse/admin/model/v1/StandardInfraConfigSpecBroker.java
@@ -91,4 +91,13 @@ public class StandardInfraConfigSpecBroker extends AbstractWithAdditionalPropert
         return Objects.hash(resources, addressFullPolicy, storageClassName, updatePersistentVolumeClaim);
     }
 
+    @Override
+    public String toString() {
+        return "StandardInfraConfigSpecBroker{" +
+                "resources=" + resources +
+                ", addressFullPolicy='" + addressFullPolicy + '\'' +
+                ", storageClassName='" + storageClassName + '\'' +
+                ", updatePersistentVolumeClaim=" + updatePersistentVolumeClaim +
+                '}';
+    }
 }
diff --git a/api-model/src/main/java/io/enmasse/admin/model/v1/StandardInfraConfigSpecBrokerResources.java b/api-model/src/main/java/io/enmasse/admin/model/v1/StandardInfraConfigSpecBrokerResources.java
index 0a781a173..acd25a2c7 100644
--- a/api-model/src/main/java/io/enmasse/admin/model/v1/StandardInfraConfigSpecBrokerResources.java
+++ b/api-model/src/main/java/io/enmasse/admin/model/v1/StandardInfraConfigSpecBrokerResources.java
@@ -63,4 +63,11 @@ public class StandardInfraConfigSpecBrokerResources extends AbstractWithAddition
         return Objects.hash(memory, storage);
     }
 
+    @Override
+    public String toString() {
+        return "StandardInfraConfigSpecBrokerResources{" +
+                "memory='" + memory + '\'' +
+                ", storage='" + storage + '\'' +
+                '}';
+    }
 }
diff --git a/api-model/src/main/java/io/enmasse/admin/model/v1/StandardInfraConfigSpecRouter.java b/api-model/src/main/java/io/enmasse/admin/model/v1/StandardInfraConfigSpecRouter.java
index 8685342ef..23d211157 100644
--- a/api-model/src/main/java/io/enmasse/admin/model/v1/StandardInfraConfigSpecRouter.java
+++ b/api-model/src/main/java/io/enmasse/admin/model/v1/StandardInfraConfigSpecRouter.java
@@ -19,7 +19,7 @@ import io.sundr.builder.annotations.Inline;
         builderPackage = "io.fabric8.kubernetes.api.builder",
         inline = @Inline(type = Doneable.class, prefix = "Doneable", value = "done")
 )
-@JsonPropertyOrder({"resources", "linkCapacity"})
+@JsonPropertyOrder({"minReplicas", "resources", "linkCapacity"})
 @JsonInclude(JsonInclude.Include.NON_NULL)
 public class StandardInfraConfigSpecRouter extends AbstractWithAdditionalProperties {
     private StandardInfraConfigSpecRouterResources resources;
@@ -72,4 +72,13 @@ public class StandardInfraConfigSpecRouter extends AbstractWithAdditionalPropert
     public int getMinReplicas() {
         return minReplicas;
     }
+
+    @Override
+    public String toString() {
+        return "StandardInfraConfigSpecRouter{" +
+                "resources=" + resources +
+                ", minReplicas=" + minReplicas +
+                ", linkCapacity=" + linkCapacity +
+                '}';
+    }
 }
diff --git a/api-model/src/main/java/io/enmasse/admin/model/v1/StandardInfraConfigSpecRouterResources.java b/api-model/src/main/java/io/enmasse/admin/model/v1/StandardInfraConfigSpecRouterResources.java
index 4fa608517..31ef5684f 100644
--- a/api-model/src/main/java/io/enmasse/admin/model/v1/StandardInfraConfigSpecRouterResources.java
+++ b/api-model/src/main/java/io/enmasse/admin/model/v1/StandardInfraConfigSpecRouterResources.java
@@ -52,4 +52,10 @@ public class StandardInfraConfigSpecRouterResources extends AbstractWithAddition
         return Objects.hash(memory);
     }
 
+    @Override
+    public String toString() {
+        return "StandardInfraConfigSpecRouterResources{" +
+                "memory='" + memory + '\'' +
+                '}';
+    }
 }
diff --git a/standard-controller/src/main/java/io/enmasse/controller/standard/AddressController.java b/standard-controller/src/main/java/io/enmasse/controller/standard/AddressController.java
index b5eadbac5..8424a2a67 100644
--- a/standard-controller/src/main/java/io/enmasse/controller/standard/AddressController.java
+++ b/standard-controller/src/main/java/io/enmasse/controller/standard/AddressController.java
@@ -41,8 +41,9 @@ public class AddressController implements Watcher<Address> {
     private final SchemaProvider schemaProvider;
     private final Vertx vertx = Vertx.vertx();
     private final Metrics metrics;
+    private final BrokerIdGenerator brokerIdGenerator;
 
-    public AddressController(StandardControllerOptions options, AddressApi addressApi, Kubernetes kubernetes, BrokerSetGenerator clusterGenerator, EventLogger eventLogger, SchemaProvider schemaProvider, Metrics metrics) {
+    public AddressController(StandardControllerOptions options, AddressApi addressApi, Kubernetes kubernetes, BrokerSetGenerator clusterGenerator, EventLogger eventLogger, SchemaProvider schemaProvider, Metrics metrics, BrokerIdGenerator brokerIdGenerator) {
         this.options = options;
         this.addressApi = addressApi;
         this.kubernetes = kubernetes;
@@ -50,6 +51,7 @@ public class AddressController implements Watcher<Address> {
         this.eventLogger = eventLogger;
         this.schemaProvider = schemaProvider;
         this.metrics = metrics;
+        this.brokerIdGenerator = brokerIdGenerator;
     }
 
     public void start() throws Exception {
@@ -92,7 +94,7 @@ public class AddressController implements Watcher<Address> {
 
         long resolvedPlan = System.nanoTime();
 
-        AddressProvisioner provisioner = new AddressProvisioner(addressSpaceResolver, addressResolver, addressSpacePlan, clusterGenerator, kubernetes, eventLogger, options.getInfraUuid());
+        AddressProvisioner provisioner = new AddressProvisioner(addressSpaceResolver, addressResolver, addressSpacePlan, clusterGenerator, kubernetes, eventLogger, options.getInfraUuid(), brokerIdGenerator);
 
         Map<Status.Phase, Long> countByPhase = countPhases(addressSet);
         log.info("Total: {}, Active: {}, Configuring: {}, Pending: {}, Terminating: {}, Failed: {}", addressSet.size(), countByPhase.get(Active), countByPhase.get(Configuring), countByPhase.get(Pending), countByPhase.get(Terminating), countByPhase.get(Failed));
@@ -207,8 +209,8 @@ public class AddressController implements Watcher<Address> {
                                 .build();
                     }
                     BrokerCluster upgradedCluster = null;
-                    if (cluster.getClusterId().startsWith("broker-pooled")) {
-                        upgradedCluster = clusterGenerator.generateCluster(cluster.getClusterId(), cluster.getNewReplicas(), null, null, desiredConfig);
+                    if (!cluster.getClusterId().startsWith("broker-sharded")) {
+                        upgradedCluster = clusterGenerator.generateCluster(cluster.getClusterId(), 1, null, null, desiredConfig);
                     } else {
                         Address address = addresses.stream()
                                 .filter(a -> cluster.getClusterId().equals(a.getAnnotation(AnnotationKeys.CLUSTER_ID)))
@@ -216,7 +218,14 @@ public class AddressController implements Watcher<Address> {
                                 .orElse(null);
                         if (address != null) {
                             AddressPlan plan = addressResolver.getPlan(address);
-                            upgradedCluster = clusterGenerator.generateCluster(cluster.getClusterId(), cluster.getNewReplicas(), address, plan, desiredConfig);
+                            int brokerNeeded = 0;
+                            for (ResourceRequest resourceRequest : plan.getRequiredResources()) {
+                                if (resourceRequest.getName().equals("broker")) {
+                                    brokerNeeded = (int) resourceRequest.getCredit();
+                                    break;
+                                }
+                            }
+                            upgradedCluster = clusterGenerator.generateCluster(cluster.getClusterId(), brokerNeeded, address, plan, desiredConfig);
                         }
                     }
                     log.info("Upgrading broker {}", cluster.getClusterId());
diff --git a/standard-controller/src/main/java/io/enmasse/controller/standard/AddressProvisioner.java b/standard-controller/src/main/java/io/enmasse/controller/standard/AddressProvisioner.java
index 6ca77ea11..bd0eee9da 100644
--- a/standard-controller/src/main/java/io/enmasse/controller/standard/AddressProvisioner.java
+++ b/standard-controller/src/main/java/io/enmasse/controller/standard/AddressProvisioner.java
@@ -10,15 +10,7 @@ import static io.enmasse.controller.standard.ControllerReason.BrokerCreated;
 import static io.enmasse.k8s.api.EventLogger.Type.Normal;
 import static io.enmasse.k8s.api.EventLogger.Type.Warning;
 
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
+import java.util.*;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import java.util.zip.CRC32;
@@ -40,8 +32,10 @@ public class AddressProvisioner {
     private final Kubernetes kubernetes;
     private final EventLogger eventLogger;
     private final String infraUuid;
+    private final String pooledClusterIdPrefix;
+    private final BrokerIdGenerator brokerIdGenerator;
 
-    public AddressProvisioner(AddressSpaceResolver addressSpaceResolver, AddressResolver addressResolver, AddressSpacePlan addressSpacePlan, BrokerSetGenerator clusterGenerator, Kubernetes kubernetes, EventLogger eventLogger, String infraUuid) {
+    public AddressProvisioner(AddressSpaceResolver addressSpaceResolver, AddressResolver addressResolver, AddressSpacePlan addressSpacePlan, BrokerSetGenerator clusterGenerator, Kubernetes kubernetes, EventLogger eventLogger, String infraUuid, BrokerIdGenerator brokerIdGenerator) {
         this.addressSpaceResolver = addressSpaceResolver;
         this.addressResolver = addressResolver;
         this.addressSpacePlan = addressSpacePlan;
@@ -49,7 +43,9 @@ public class AddressProvisioner {
         this.kubernetes = kubernetes;
         this.eventLogger = eventLogger;
         this.infraUuid = infraUuid;
-        this.pooledPattern = Pattern.compile("^broker-pooled-" + infraUuid + "-\\d+");
+        this.pooledPattern = Pattern.compile("^broker-" + infraUuid + "-.*");
+        this.pooledClusterIdPrefix = "broker-" + infraUuid + "-";
+        this.brokerIdGenerator = brokerIdGenerator;
     }
 
     /**
@@ -74,20 +70,20 @@ public class AddressProvisioner {
             if ("subscription".equals(address.getType())) {
                 resourceName = "subscription";
                 if (!getBrokerId(address).isPresent()) {
-                    log.warn("Unexpected pooled address without broker id: " + address.getAddress());
+                    log.warn("Unexpected subscription without broker id: " + address.getAddress());
                     return;
                 }
-                instanceId = getBrokerId(address).orElseThrow(() -> new IllegalArgumentException("Unexpected pooled address without broker id: " + address.getAddress()));
+                instanceId = getBrokerId(address).get();
             } else if (resourceRequest.getName().equals("router")) {
                 instanceId = "all";
 
             //Should we check the plan type, instead of the amount? addressPlan.getName="pooled-topic"
             } else if (resourceRequest.getName().equals("broker") && resourceRequest.getCredit() < 1) {
-                if (!getBrokerId(address).isPresent()) {
-                    log.warn("Unexpected pooled address without broker id: " + address.getAddress());
+                if (!getClusterId(address).isPresent()) {
+                    log.warn("Unexpected pooled address without cluster id: " + address.getAddress());
                     return;
                 }
-                instanceId = getBrokerId(address).orElseThrow(() -> new IllegalArgumentException("Unexpected pooled address without broker id: " + address.getAddress()));
+                instanceId = getClusterId(address).get();
 
             //Should we check the plan type, instead of the amount? addressPlan.getName="sharded-topic"
             } else if (resourceRequest.getName().equals("broker")) {
@@ -178,7 +174,7 @@ public class AddressProvisioner {
         String cluster = topic.getAnnotation(AnnotationKeys.CLUSTER_ID);
         String broker = topic.getAnnotation(AnnotationKeys.BROKER_ID);
 
-        boolean isPooled = broker !=null;
+        boolean isPooled = broker != null;
 
         subscription.putAnnotation(AnnotationKeys.CLUSTER_ID, cluster);
 
@@ -244,7 +240,7 @@ public class AddressProvisioner {
                 } else if (resourceRequest.getCredit() < 1) {
                     boolean scheduled = scheduleAddress(resourceUsage, address, resourceRequest.getCredit());
                     if (!scheduled) {
-                        allocateBroker(resourceUsage, "broker-pooled-" + infraUuid + "-");
+                        allocateBroker(resourceUsage, pooledClusterIdPrefix);
                         if (!scheduleAddress(resourceUsage, address, resourceRequest.getCredit())) {
                             log.warn("Unable to find broker for scheduling {}", address);
                             return null;
@@ -349,6 +345,10 @@ public class AddressProvisioner {
         return "broker-sharded-" + Long.toHexString(crc32.getValue()) + "-" + infraUuid;
     }
 
+    public static Optional<String> getClusterId(Address address) {
+        return Optional.ofNullable(address.getAnnotation(AnnotationKeys.CLUSTER_ID));
+    }
+
     public static Optional<String> getBrokerId(Address address) {
         return Optional.ofNullable(address.getAnnotation(AnnotationKeys.BROKER_ID));
     }
@@ -379,7 +379,11 @@ public class AddressProvisioner {
                 // Provision pooled broker
                 int needPooled = sumNeededMatching(entry.getValue(), pooledPattern);
                 if (needPooled > 0) {
-                    provisionBroker(existingClusters, "broker-pooled-" + infraUuid, needPooled, null, null);
+                    for (String id : entry.getValue().keySet()) {
+                        if (pooledPattern.matcher(id).matches()) {
+                            provisionBroker(existingClusters, id, 1, null, null);
+                        }
+                    }
                 }
 
                 // Collect all sharded brokers
@@ -406,19 +410,11 @@ public class AddressProvisioner {
             log.info("Scaling router to {} replicas", router.getNewReplicas());
             kubernetes.scaleStatefulSet(router.getName(), router.getNewReplicas());
         }
-        for (BrokerCluster cluster : existingClusters) {
-            if (cluster.hasChanged()) {
-                log.info("Scaling broker cluster {} to {} replicas", cluster.getClusterId(), cluster.getNewReplicas());
-                kubernetes.scaleStatefulSet(cluster.getClusterId(), cluster.getNewReplicas());
-            }
-        }
     }
 
     private final Pattern pooledPattern;
     private boolean scheduleAddress(Map<String, UsageInfo> usageMap, Address address, double credit) {
 
-        address.putAnnotation(AnnotationKeys.CLUSTER_ID, "broker-pooled-" + infraUuid);
-
         List<BrokerInfo> brokers = new ArrayList<>();
         for (String host : usageMap.keySet()) {
             if (pooledPattern.matcher(host).matches()) {
@@ -430,7 +426,8 @@ public class AddressProvisioner {
 
         for (BrokerInfo brokerInfo : brokers) {
             if (brokerInfo.getCredit() + credit < 1) {
-                address.putAnnotation(AnnotationKeys.BROKER_ID, brokerInfo.getBrokerId());
+                address.putAnnotation(AnnotationKeys.BROKER_ID, brokerInfo.getBrokerId() + "-0");
+                address.putAnnotation(AnnotationKeys.CLUSTER_ID, brokerInfo.getBrokerId());
                 UsageInfo used = usageMap.get(brokerInfo.getBrokerId());
                 used.addUsed(credit);
                 return true;
@@ -439,22 +436,17 @@ public class AddressProvisioner {
         return false;
     }
 
-    private void allocateBroker(Map<String, UsageInfo> resourceNeeded, String clusterId) {
-        int numPooled = 0;
-        for (String id : resourceNeeded.keySet()) {
-            if (pooledPattern.matcher(id).matches()) {
-                numPooled++;
-            }
-        }
-
-        resourceNeeded.put(clusterId + numPooled, new UsageInfo());
+    private String allocateBroker(Map<String, UsageInfo> resourceNeeded, String clusterIdPrefix) {
+        String randomId = brokerIdGenerator.generateBrokerId();
+        String clusterId = clusterIdPrefix + randomId;
+        resourceNeeded.put(clusterId, new UsageInfo());
+        return clusterId;
     }
 
     private void provisionBroker(List<BrokerCluster> clusterList, String clusterId, int numReplicas, Address address, AddressPlan addressPlan) {
         try {
             for (BrokerCluster cluster : clusterList) {
                 if (cluster.getClusterId().equals(clusterId)) {
-                    cluster.setNewReplicas(numReplicas);
                     return;
                 }
             }
@@ -493,4 +485,5 @@ public class AddressProvisioner {
         }
 
     }
+
 }
diff --git a/standard-controller/src/main/java/io/enmasse/controller/standard/BrokerCluster.java b/standard-controller/src/main/java/io/enmasse/controller/standard/BrokerCluster.java
index 6ac559236..5e9061d1f 100644
--- a/standard-controller/src/main/java/io/enmasse/controller/standard/BrokerCluster.java
+++ b/standard-controller/src/main/java/io/enmasse/controller/standard/BrokerCluster.java
@@ -49,26 +49,11 @@ public class BrokerCluster {
     }
 
     private final String clusterId;
-    private final int replicas;
     private KubernetesList resources;
-    private int newReplicas;
 
     public BrokerCluster(String clusterId, KubernetesList resources) {
         this.clusterId = clusterId;
         this.resources = resources;
-        this.replicas = findReplicas(resources.getItems());
-        this.newReplicas = replicas;
-    }
-
-    private int findReplicas(List<HasMetadata> items) {
-        for (HasMetadata item : items) {
-            if (item instanceof StatefulSet) {
-                return ((StatefulSet)item).getSpec().getReplicas();
-            } else if (item instanceof Deployment) {
-                return ((Deployment)item).getSpec().getReplicas();
-            }
-        }
-        return 0;
     }
 
     private StandardInfraConfig findStandardInfraConfig(List<HasMetadata> items) throws IOException {
@@ -84,23 +69,6 @@ public class BrokerCluster {
         return config;
     }
 
-
-    public void setNewReplicas(int replicas) {
-        this.newReplicas = replicas;
-    }
-
-    public int getReplicas() {
-        return replicas;
-    }
-
-    public int getNewReplicas() {
-        return newReplicas;
-    }
-
-    public boolean hasChanged() {
-        return replicas != newReplicas;
-    }
-
     public KubernetesList getResources() {
         return resources;
     }
diff --git a/standard-controller/src/main/java/io/enmasse/controller/standard/BrokerIdGenerator.java b/standard-controller/src/main/java/io/enmasse/controller/standard/BrokerIdGenerator.java
new file mode 100644
index 000000000..2122e27cf
--- /dev/null
+++ b/standard-controller/src/main/java/io/enmasse/controller/standard/BrokerIdGenerator.java
@@ -0,0 +1,9 @@
+/*
+ * Copyright 2018, EnMasse authors.
+ * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html).
+ */
+package io.enmasse.controller.standard;
+
+public interface BrokerIdGenerator {
+    String generateBrokerId();
+}
diff --git a/standard-controller/src/main/java/io/enmasse/controller/standard/RandomBrokerIdGenerator.java b/standard-controller/src/main/java/io/enmasse/controller/standard/RandomBrokerIdGenerator.java
new file mode 100644
index 000000000..64dbec779
--- /dev/null
+++ b/standard-controller/src/main/java/io/enmasse/controller/standard/RandomBrokerIdGenerator.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2018, EnMasse authors.
+ * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html).
+ */
+package io.enmasse.controller.standard;
+
+import java.util.Random;
+
+public class RandomBrokerIdGenerator implements BrokerIdGenerator {
+    private final Random random = new Random(System.currentTimeMillis());
+    private static final String ALPHA_NUMERIC_STRING = "abcdefghijklmnopqrstuvwxyz0123456789";
+
+    public String generateBrokerId() {
+        return generateAlphaNumberic(4);
+    }
+
+    private String generateAlphaNumberic(int length) {
+        StringBuilder builder = new StringBuilder();
+        while (length-- != 0) {
+            int character = (int)(random.nextDouble()*ALPHA_NUMERIC_STRING.length());
+            builder.append(ALPHA_NUMERIC_STRING.charAt(character));
+        }
+        return builder.toString();
+    }
+}
diff --git a/standard-controller/src/main/java/io/enmasse/controller/standard/StandardController.java b/standard-controller/src/main/java/io/enmasse/controller/standard/StandardController.java
index eaf5ecee9..590244765 100644
--- a/standard-controller/src/main/java/io/enmasse/controller/standard/StandardController.java
+++ b/standard-controller/src/main/java/io/enmasse/controller/standard/StandardController.java
@@ -93,7 +93,8 @@ public class StandardController {
                 clusterGenerator,
                 eventLogger,
                 schemaProvider,
-                metrics);
+                metrics,
+                new RandomBrokerIdGenerator());
 
         log.info("Starting standard controller for " + options.getAddressSpace());
         addressController.start();


Test patch:
diff --git a/address-model-lib/src/test/java/io/enmasse/address/model/AuthenticationServiceTest.java b/api-model/src/test/java/io/enmasse/address/model/AuthenticationServiceTest.java
similarity index 100%
rename from address-model-lib/src/test/java/io/enmasse/address/model/AuthenticationServiceTest.java
rename to api-model/src/test/java/io/enmasse/address/model/AuthenticationServiceTest.java
diff --git a/api-model/src/test/java/io/enmasse/address/model/v1/address/SerializationTest.java b/api-model/src/test/java/io/enmasse/address/model/v1/address/SerializationTest.java
index d51393e21..cc4ba2da5 100644
--- a/api-model/src/test/java/io/enmasse/address/model/v1/address/SerializationTest.java
+++ b/api-model/src/test/java/io/enmasse/address/model/v1/address/SerializationTest.java
@@ -7,10 +7,10 @@ package io.enmasse.address.model.v1.address;
 import io.enmasse.address.model.*;
 import io.enmasse.address.model.v1.CodecV1;
 import io.enmasse.address.model.v1.DeserializeException;
-import io.enmasse.admin.model.v1.AddressPlan;
-import io.enmasse.admin.model.v1.AddressSpacePlan;
-import io.enmasse.admin.model.v1.AddressSpacePlanBuilder;
+import io.enmasse.admin.model.v1.*;
 import io.fabric8.kubernetes.api.model.ObjectMetaBuilder;
+import io.fabric8.kubernetes.api.model.networking.NetworkPolicyIngressRule;
+import io.fabric8.kubernetes.api.model.networking.NetworkPolicyIngressRuleBuilder;
 import org.junit.jupiter.api.Test;
 import org.mockito.internal.util.collections.Sets;
 
@@ -470,6 +470,152 @@ public class SerializationTest {
         assertAddressSpace(deserialized, a2);
     }
 
+    @Test
+    public void testSerializeStandardInfraConfig() throws IOException {
+        StandardInfraConfig infraConfig = new StandardInfraConfigBuilder()
+                .withMetadata(new ObjectMetaBuilder()
+                        .withName("infra")
+                        .build())
+                .editOrNewSpec()
+                .withVersion("123")
+                .editOrNewNetworkPolicy()
+                .withIngress(new NetworkPolicyIngressRuleBuilder().build())
+                .endNetworkPolicy()
+                .editOrNewAdmin()
+                .editOrNewResources()
+                .withMemory("512Mi")
+                .endResources()
+                .endAdmin()
+                .editOrNewBroker()
+                .editOrNewResources()
+                .withMemory("128Mi")
+                .withStorage("2Gi")
+                .endResources()
+                .withStorageClassName("local")
+                .withUpdatePersistentVolumeClaim(false)
+                .withAddressFullPolicy("FAIL")
+                .endBroker()
+                .editOrNewRouter()
+                .editOrNewResources()
+                .withMemory("128Mi")
+                .endResources()
+                .withLinkCapacity(100)
+                .endRouter()
+                .endSpec()
+                .build();
+
+        String serialized = CodecV1.getMapper().writeValueAsString(infraConfig);
+        StandardInfraConfig deserialized = CodecV1.getMapper().readValue(serialized, StandardInfraConfig.class);
+        assertEquals(infraConfig, deserialized);
+
+        serialized = "{" +
+                "\"apiVersion\":\"admin.enmasse.io/v1alpha1\"," +
+                "\"kind\":\"StandardInfraConfig\"," +
+                "\"metadata\":{" +
+                "  \"name\":\"infra\"," +
+                "  \"labels\": {}," +
+                "  \"annotations\": {}" +
+                "}," +
+                "\"spec\": {" +
+                "  \"version\": \"123\"," +
+                "  \"networkPolicy\": {" +
+                "    \"ingress\": [{\"from\":[],\"ports\":[]}]," +
+                "    \"egress\": []" +
+                "  }," +
+                "  \"broker\": {" +
+                "     \"resources\": {" +
+                "       \"memory\": \"128Mi\"," +
+                "       \"storage\": \"2Gi\"" +
+                "     }," +
+                "     \"addressFullPolicy\": \"FAIL\"," +
+                "     \"storageClassName\": \"local\"," +
+                "     \"updatePersistentVolumeClaim\": false" +
+                "  }," +
+                "  \"admin\": {" +
+                "     \"resources\": {" +
+                "       \"memory\": \"512Mi\"" +
+                "     }" +
+                "  }," +
+                "  \"router\": {" +
+                "     \"resources\": {" +
+                "       \"memory\": \"128Mi\"" +
+                "     }," +
+                "     \"linkCapacity\": 100" +
+                "  }" +
+                "}}";
+
+
+        deserialized = CodecV1.getMapper().readValue(serialized, StandardInfraConfig.class);
+        assertEquals(infraConfig, deserialized);
+    }
+
+    @Test
+    public void testSerializeBrokeredInfraConfig() throws IOException {
+        BrokeredInfraConfig infraConfig = new BrokeredInfraConfigBuilder()
+                .withMetadata(new ObjectMetaBuilder()
+                        .withName("infra")
+                        .build())
+                .editOrNewSpec()
+                .withVersion("123")
+                .editOrNewNetworkPolicy()
+                .withIngress(new NetworkPolicyIngressRuleBuilder().build())
+                .endNetworkPolicy()
+                .editOrNewAdmin()
+                .editOrNewResources()
+                .withMemory("512Mi")
+                .endResources()
+                .endAdmin()
+                .editOrNewBroker()
+                .editOrNewResources()
+                .withMemory("128Mi")
+                .withStorage("2Gi")
+                .endResources()
+                .withStorageClassName("local")
+                .withUpdatePersistentVolumeClaim(false)
+                .withAddressFullPolicy("FAIL")
+                .endBroker()
+                .endSpec()
+                .build();
+
+        String serialized = CodecV1.getMapper().writeValueAsString(infraConfig);
+        BrokeredInfraConfig deserialized = CodecV1.getMapper().readValue(serialized, BrokeredInfraConfig.class);
+        assertEquals(infraConfig, deserialized);
+
+        serialized = "{" +
+                "\"apiVersion\":\"admin.enmasse.io/v1alpha1\"," +
+                "\"kind\":\"BrokeredInfraConfig\"," +
+                "\"metadata\":{" +
+                "  \"name\":\"infra\"," +
+                "  \"labels\": {}," +
+                "  \"annotations\": {}" +
+                "}," +
+                "\"spec\": {" +
+                "  \"version\": \"123\"," +
+                "  \"networkPolicy\": {" +
+                "    \"ingress\": [{\"from\":[],\"ports\":[]}]," +
+                "    \"egress\": []" +
+                "  }," +
+                "  \"broker\": {" +
+                "     \"resources\": {" +
+                "       \"memory\": \"128Mi\"," +
+                "       \"storage\": \"2Gi\"" +
+                "     }," +
+                "     \"addressFullPolicy\": \"FAIL\"," +
+                "     \"storageClassName\": \"local\"," +
+                "     \"updatePersistentVolumeClaim\": false" +
+                "  }," +
+                "  \"admin\": {" +
+                "     \"resources\": {" +
+                "       \"memory\": \"512Mi\"" +
+                "     }" +
+                "  }" +
+                "}}";
+
+
+        deserialized = CodecV1.getMapper().readValue(serialized, BrokeredInfraConfig.class);
+        assertEquals(infraConfig, deserialized);
+    }
+
     private void assertAddressSpace(AddressSpaceList deserialized, AddressSpace expected) {
         AddressSpace found = null;
         for (AddressSpace addressSpace : deserialized) {
diff --git a/standard-controller/src/test/java/io/enmasse/controller/standard/AddressControllerTest.java b/standard-controller/src/test/java/io/enmasse/controller/standard/AddressControllerTest.java
index 3534d52a9..25b97da85 100644
--- a/standard-controller/src/test/java/io/enmasse/controller/standard/AddressControllerTest.java
+++ b/standard-controller/src/test/java/io/enmasse/controller/standard/AddressControllerTest.java
@@ -30,9 +30,12 @@ public class AddressControllerTest {
     private AddressController controller;
     private OpenShiftClient mockClient;
     private BrokerSetGenerator mockGenerator;
+    private int id = 0;
+    private BrokerIdGenerator idGenerator = () -> String.valueOf(id++);
 
     @BeforeEach
     public void setUp() throws IOException {
+        id = 0;
         mockHelper = mock(Kubernetes.class);
         mockGenerator = mock(BrokerSetGenerator.class);
         mockApi = mock(AddressApi.class);
@@ -45,7 +48,7 @@ public class AddressControllerTest {
         options.setAddressSpacePlanName("plan1");
         options.setResyncInterval(Duration.ofSeconds(5));
         options.setVersion("1.0");
-        controller = new AddressController(options, mockApi, mockHelper, mockGenerator, eventLogger, standardControllerSchema, new Metrics());
+        controller = new AddressController(options, mockApi, mockHelper, mockGenerator, eventLogger, standardControllerSchema, new Metrics(), idGenerator);
     }
 
     @Test
diff --git a/standard-controller/src/test/java/io/enmasse/controller/standard/AddressProvisionerTest.java b/standard-controller/src/test/java/io/enmasse/controller/standard/AddressProvisionerTest.java
index 6402e4499..ae74870fe 100644
--- a/standard-controller/src/test/java/io/enmasse/controller/standard/AddressProvisionerTest.java
+++ b/standard-controller/src/test/java/io/enmasse/controller/standard/AddressProvisionerTest.java
@@ -35,9 +35,12 @@ import static org.mockito.Mockito.*;
 public class AddressProvisionerTest {
     private BrokerSetGenerator generator;
     private Kubernetes kubernetes;
+    private int id = 0;
+    private BrokerIdGenerator idGenerator = () -> String.valueOf(id++);
 
     @BeforeEach
     public void setup() {
+        id = 0;
         generator = mock(BrokerSetGenerator.class);
         kubernetes = mock(Kubernetes.class);
     }
@@ -48,7 +51,7 @@ public class AddressProvisionerTest {
         AddressSpaceResolver addressSpaceResolver = new AddressSpaceResolver(standardControllerSchema.getSchema());
         EventLogger logger = mock(EventLogger.class);
 
-        return new AddressProvisioner(addressSpaceResolver, resolver, standardControllerSchema.getPlan(), generator, kubernetes, logger, "1234");
+        return new AddressProvisioner(addressSpaceResolver, resolver, standardControllerSchema.getPlan(), generator, kubernetes, logger, "1234", idGenerator);
     }
 
     private AddressProvisioner createProvisioner(List<ResourceAllowance> resourceAllowances) {
@@ -57,7 +60,7 @@ public class AddressProvisionerTest {
         AddressSpaceResolver addressSpaceResolver = new AddressSpaceResolver(standardControllerSchema.getSchema());
         EventLogger logger = mock(EventLogger.class);
 
-        return new AddressProvisioner(addressSpaceResolver, resolver, standardControllerSchema.getPlan(), generator, kubernetes, logger, "1234");
+        return new AddressProvisioner(addressSpaceResolver, resolver, standardControllerSchema.getPlan(), generator, kubernetes, logger, "1234", idGenerator);
     }
 
     @Test
@@ -83,7 +86,8 @@ public class AddressProvisionerTest {
                 .setNamespace("ns")
                 .setPlan("small-queue")
                 .setType("queue")
-                .putAnnotation(AnnotationKeys.BROKER_ID, "broker-0")
+                .putAnnotation(AnnotationKeys.BROKER_ID, "broker-0-0")
+                .putAnnotation(AnnotationKeys.CLUSTER_ID, "broker-0")
                 .build());
 
         usageMap = provisioner.checkUsage(addresses);
@@ -100,7 +104,8 @@ public class AddressProvisionerTest {
                 .setNamespace("ns")
                 .setPlan("small-queue")
                 .setType("queue")
-                .putAnnotation(AnnotationKeys.BROKER_ID, "broker-0")
+                .putAnnotation(AnnotationKeys.BROKER_ID, "broker-0-0")
+                .putAnnotation(AnnotationKeys.CLUSTER_ID, "broker-0")
                 .build());
 
         usageMap = provisioner.checkUsage(addresses);
@@ -115,9 +120,9 @@ public class AddressProvisionerTest {
     @Test
     public void testQuotaCheck() {
         Set<Address> addresses = new HashSet<>();
-        addresses.add(createQueue("q1", "small-queue").putAnnotation(AnnotationKeys.BROKER_ID, "broker-pooled-1234-0"));
-        addresses.add(createQueue("q2", "small-queue").putAnnotation(AnnotationKeys.BROKER_ID, "broker-pooled-1234-0"));
-        addresses.add(createQueue("q3", "small-queue").putAnnotation(AnnotationKeys.BROKER_ID, "broker-pooled-1234-1"));
+        addresses.add(createQueue("q1", "small-queue").putAnnotation(AnnotationKeys.CLUSTER_ID, "broker-1234-0").putAnnotation(AnnotationKeys.BROKER_ID, "broker-1234-0-0"));
+        addresses.add(createQueue("q2", "small-queue").putAnnotation(AnnotationKeys.CLUSTER_ID, "broker-1234-0").putAnnotation(AnnotationKeys.BROKER_ID, "broker-1234-0-0"));
+        addresses.add(createQueue("q3", "small-queue").putAnnotation(AnnotationKeys.CLUSTER_ID, "broker-1234-1").putAnnotation(AnnotationKeys.BROKER_ID, "broker-1234-1-0"));
 
         AddressProvisioner provisioner = createProvisioner();
         Map<String, Map<String, UsageInfo>> usageMap = provisioner.checkUsage(addresses);
@@ -161,7 +166,7 @@ public class AddressProvisionerTest {
     public void testProvisioningColocated() {
         Set<Address> addresses = new HashSet<>();
         addresses.add(createAddress("a1", "anycast", "small-anycast"));
-        addresses.add(createAddress("q1", "queue", "small-queue").putAnnotation(AnnotationKeys.BROKER_ID, "broker-0"));
+        addresses.add(createAddress("q1", "queue", "small-queue").putAnnotation(AnnotationKeys.BROKER_ID, "broker-1234-0"));
 
 
         AddressProvisioner provisioner = createProvisioner();
@@ -170,13 +175,13 @@ public class AddressProvisionerTest {
         Address queue = createAddress("q2", "queue", "small-queue");
         Map<String, Map<String, UsageInfo>> neededMap = provisioner.checkQuota(usageMap, Sets.newSet(queue), Sets.newSet(queue));
 
-        List<BrokerCluster> clusterList = Arrays.asList(new BrokerCluster("broker-pooled-1234", new KubernetesList()));
+        List<BrokerCluster> clusterList = Arrays.asList(new BrokerCluster("broker-1234-0", new KubernetesList()));
         provisioner.provisionResources(createDeployment(1), clusterList, neededMap, Sets.newSet(queue));
 
         assertThat(clusterList.get(0).getResources().getItems().size(), is(0));
         assertTrue(queue.getStatus().getMessages().isEmpty(), queue.getStatus().getMessages().toString());
         assertThat(queue.getStatus().getPhase(), is(Status.Phase.Configuring));
-        assertThat(queue.getAnnotations().get(AnnotationKeys.BROKER_ID), is("broker-pooled-1234-0"));
+        assertThat(queue.getAnnotations().get(AnnotationKeys.BROKER_ID), is("broker-1234-0-0"));
     }
 
     private static RouterCluster createDeployment(int replicas) {
@@ -184,11 +189,12 @@ public class AddressProvisionerTest {
     }
 
     @Test
-    public void testScalingColocated() {
+    public void testScalingColocated() throws Exception {
         Set<Address> addresses = new HashSet<>();
         addresses.add(createAddress("a1", "anycast", "small-anycast"));
-        addresses.add(createAddress("q1", "queue", "small-queue").putAnnotation(AnnotationKeys.BROKER_ID, "broker-pooled-1234-0"));
-        addresses.add(createAddress("q2", "queue", "small-queue").putAnnotation(AnnotationKeys.BROKER_ID, "broker-pooled-1234-0"));
+        addresses.add(createAddress("q1", "queue", "small-queue").putAnnotation(AnnotationKeys.CLUSTER_ID, "broker-1234-0"));
+        addresses.add(createAddress("q2", "queue", "small-queue").putAnnotation(AnnotationKeys.CLUSTER_ID, "broker-1234-0"));
+        id = 1;
 
         AddressProvisioner provisioner = createProvisioner();
         Map<String, Map<String, UsageInfo>> usageMap = provisioner.checkUsage(addresses);
@@ -196,13 +202,14 @@ public class AddressProvisionerTest {
         Address queue = createAddress("q3", "queue", "small-queue");
         Map<String, Map<String, UsageInfo>> provisionMap = provisioner.checkQuota(usageMap, Sets.newSet(queue), Sets.newSet(queue));
 
-        List<BrokerCluster> clusterList = Arrays.asList(new BrokerCluster("broker-pooled-1234", new KubernetesList()));
+        List<BrokerCluster> clusterList = new ArrayList<>();
+        clusterList.add(new BrokerCluster("broker-1234-0", new KubernetesList()));
+        when(generator.generateCluster(eq("broker-1234-1"), anyInt(), any(), any(), any())).thenReturn(new BrokerCluster("broker-1234-1", new KubernetesList()));
         provisioner.provisionResources(createDeployment(1), clusterList, provisionMap, Sets.newSet(queue));
-        verify(kubernetes).scaleStatefulSet(eq("broker-pooled-1234"), eq(2));
 
         assertTrue(queue.getStatus().getMessages().isEmpty(), queue.getStatus().getMessages().toString());
         assertThat(queue.getStatus().getPhase(), is(Status.Phase.Configuring));
-        assertThat(queue.getAnnotations().get(AnnotationKeys.BROKER_ID), is("broker-pooled-1234-1"));
+        assertThat(queue.getAnnotations().get(AnnotationKeys.BROKER_ID), is("broker-1234-1-0"));
     }
 
     @Test
@@ -233,7 +240,8 @@ public class AddressProvisionerTest {
         assertThat(AddressProvisioner.sumTotalNeeded(neededMap), is(2));
 
         List<BrokerCluster> brokerClusters = Arrays.asList(
-                createCluster("broker-pooled-1234", 2));
+                createCluster("broker-1234-0", 1),
+                createCluster("broker-1234-1", 1));
 
         provisioner.provisionResources(new RouterCluster("router", 1, null), brokerClusters, neededMap, addressSet);
 
diff --git a/systemtests/src/main/java/io/enmasse/systemtest/Address.java b/systemtests/src/main/java/io/enmasse/systemtest/Address.java
index baaf00e4c..cf11d79d3 100644
--- a/systemtests/src/main/java/io/enmasse/systemtest/Address.java
+++ b/systemtests/src/main/java/io/enmasse/systemtest/Address.java
@@ -5,11 +5,13 @@
 package io.enmasse.systemtest;
 
 import java.util.List;
+import java.util.Map;
 
 public class Address {
 
     private String addressSpace;
     private String address;
+    private Map<String, Object> annotations;
     private String name;
     private String uuid;
     private String type;
@@ -19,9 +21,10 @@ public class Address {
     private List<String> statusMessages;
 
 
-    public Address(String addressSpace, String address, String name, String type, String plan, String phase, boolean isReady, List<String> statusMessages, String uuid) {
+    public Address(String addressSpace, String address, Map<String, Object> annotations, String name, String type, String plan, String phase, boolean isReady, List<String> statusMessages, String uuid) {
         this.addressSpace = addressSpace;
         this.address = address;
+        this.annotations = annotations;
         this.name = name;
         this.uuid = uuid;
         this.type = type;
@@ -66,4 +69,8 @@ public class Address {
     public String getUuid() {
         return uuid;
     }
+
+    public Map<String, Object> getAnnotations() {
+        return annotations;
+    }
 }
diff --git a/systemtests/src/main/java/io/enmasse/systemtest/AddressStatus.java b/systemtests/src/main/java/io/enmasse/systemtest/AddressStatus.java
index 6730e6eea..226cc90a5 100644
--- a/systemtests/src/main/java/io/enmasse/systemtest/AddressStatus.java
+++ b/systemtests/src/main/java/io/enmasse/systemtest/AddressStatus.java
@@ -9,5 +9,5 @@ public enum AddressStatus {
     PENDING,
     ERROR,
     WARNING,
-    UNKNOWN;
+    UNKNOWN
 }
diff --git a/systemtests/src/main/java/io/enmasse/systemtest/Endpoint.java b/systemtests/src/main/java/io/enmasse/systemtest/Endpoint.java
index 84432ee96..3e713a178 100644
--- a/systemtests/src/main/java/io/enmasse/systemtest/Endpoint.java
+++ b/systemtests/src/main/java/io/enmasse/systemtest/Endpoint.java
@@ -5,7 +5,6 @@
 
 package io.enmasse.systemtest;
 
-import java.net.MalformedURLException;
 import java.net.URL;
 
 public class Endpoint {
diff --git a/systemtests/src/main/java/io/enmasse/systemtest/GlobalLogCollector.java b/systemtests/src/main/java/io/enmasse/systemtest/GlobalLogCollector.java
index fce1ffccc..210672ca1 100644
--- a/systemtests/src/main/java/io/enmasse/systemtest/GlobalLogCollector.java
+++ b/systemtests/src/main/java/io/enmasse/systemtest/GlobalLogCollector.java
@@ -119,7 +119,7 @@ public class GlobalLogCollector {
     public void collectRouterState(String operation) {
         log.info("Collecting router state in namespace {}", namespace);
         long timestamp = System.currentTimeMillis();
-        kubernetes.listPods(Collections.singletonMap("capability", "router")).stream().forEach(pod -> {
+        kubernetes.listPods(Collections.singletonMap("capability", "router")).forEach(pod -> {
             collectRouterInfo(pod, "." + operation + ".autolinks." + timestamp, "qdmanage", "QUERY", "--type=autoLink");
             collectRouterInfo(pod, "." + operation + ".links." + timestamp, "qdmanage", "QUERY", "--type=link");
             collectRouterInfo(pod, "." + operation + ".connections." + timestamp, "qdmanage", "QUERY", "--type=connection");
@@ -130,7 +130,7 @@ public class GlobalLogCollector {
         });
     }
 
-    private void collectRouterInfo(Pod pod, String filesuffix, String ... command) {
+    private void collectRouterInfo(Pod pod, String filesuffix, String... command) {
         String output = kubernetes.runOnPod(pod, "router", command);
         try {
             Path path = Paths.get(logDir.getPath(), namespace);
diff --git a/systemtests/src/main/java/io/enmasse/systemtest/Kubernetes.java b/systemtests/src/main/java/io/enmasse/systemtest/Kubernetes.java
index b26a58c01..420e7ce17 100644
--- a/systemtests/src/main/java/io/enmasse/systemtest/Kubernetes.java
+++ b/systemtests/src/main/java/io/enmasse/systemtest/Kubernetes.java
@@ -4,8 +4,6 @@
  */
 package io.enmasse.systemtest;
 
-import io.enmasse.systemtest.executor.Executor;
-import io.enmasse.systemtest.resources.*;
 import io.fabric8.kubernetes.api.model.*;
 import io.fabric8.kubernetes.api.model.apps.Deployment;
 import io.fabric8.kubernetes.api.model.apps.StatefulSet;
@@ -15,19 +13,16 @@ import io.fabric8.kubernetes.client.Watcher;
 import io.fabric8.kubernetes.client.dsl.ExecListener;
 import io.fabric8.kubernetes.client.dsl.ExecWatch;
 import io.fabric8.kubernetes.client.dsl.LogWatch;
-import io.vertx.core.json.JsonArray;
-import io.vertx.core.json.JsonObject;
 import okhttp3.Response;
 import org.slf4j.Logger;
 
-import java.io.*;
-import java.net.MalformedURLException;
+import java.io.ByteArrayOutputStream;
+import java.io.OutputStream;
+import java.io.UnsupportedEncodingException;
 import java.nio.charset.StandardCharsets;
 import java.util.*;
 import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
 
 public abstract class Kubernetes {
@@ -149,12 +144,9 @@ public abstract class Kubernetes {
     public List<Pod> listPods(Map<String, String> labelSelector, Map<String, String> annotationSelector) {
         return client.pods().withLabels(labelSelector).list().getItems().stream().filter(pod -> {
             for (Map.Entry<String, String> entry : annotationSelector.entrySet()) {
-                if (pod.getMetadata().getAnnotations() == null
-                        || pod.getMetadata().getAnnotations().get(entry.getKey()) == null
-                        || !pod.getMetadata().getAnnotations().get(entry.getKey()).equals(entry.getValue())) {
-                    return false;
-                }
-                return true;
+                return pod.getMetadata().getAnnotations() != null
+                        && pod.getMetadata().getAnnotations().get(entry.getKey()) != null
+                        && pod.getMetadata().getAnnotations().get(entry.getKey()).equals(entry.getValue());
             }
             return true;
         }).collect(Collectors.toList());
@@ -197,7 +189,7 @@ public abstract class Kubernetes {
         if (secret == null) {
             throw new IllegalStateException("Unable to find CA cert for keycloak");
         }
-        return new String(Base64.getDecoder().decode(secret.getData().get("tls.crt")), "UTF-8");
+        return new String(Base64.getDecoder().decode(secret.getData().get("tls.crt")), StandardCharsets.UTF_8);
     }
 
     public List<ConfigMap> listConfigMaps(String type) {
@@ -253,6 +245,11 @@ public abstract class Kubernetes {
         client.namespaces().withName(namespace).delete();
     }
 
+    public void deletePod(String namespace, Map<String, String> labels) {
+        log.info("Delete pods with labels: {}", labels.toString());
+        client.pods().inNamespace(namespace).withLabels(labels).delete();
+    }
+
     /***
      * Creates pod from resources
      * @param namespace
@@ -274,7 +271,7 @@ public abstract class Kubernetes {
      * @param podName
      * @throws Exception
      */
-    public void deletePod(String namespace, String podName) throws Exception {
+    public void deletePod(String namespace, String podName) {
         client.pods().inNamespace(namespace).withName(podName).delete();
         log.info("Pod {} removed", podName);
     }
@@ -372,7 +369,7 @@ public abstract class Kubernetes {
         return listPods().get(0).getMetadata().getLabels().get("app");
     }
 
-    public String runOnPod(Pod pod, String container, String ... command) {
+    public String runOnPod(Pod pod, String container, String... command) {
         ByteArrayOutputStream baos = new ByteArrayOutputStream();
         log.info("Running command on pod {}: {}", pod.getMetadata().getName(), command);
         CompletableFuture<String> data = new CompletableFuture<>();
diff --git a/systemtests/src/main/java/io/enmasse/systemtest/Minikube.java b/systemtests/src/main/java/io/enmasse/systemtest/Minikube.java
index 361efe7b8..f5f25b55c 100644
--- a/systemtests/src/main/java/io/enmasse/systemtest/Minikube.java
+++ b/systemtests/src/main/java/io/enmasse/systemtest/Minikube.java
@@ -4,13 +4,11 @@
  */
 package io.enmasse.systemtest;
 
-import io.fabric8.kubernetes.client.ConfigBuilder;
 import io.fabric8.kubernetes.client.DefaultKubernetesClient;
 
 import java.io.BufferedReader;
 import java.io.InputStream;
 import java.io.InputStreamReader;
-import java.net.MalformedURLException;
 import java.util.concurrent.TimeUnit;
 
 public class Minikube extends Kubernetes {
diff --git a/systemtests/src/main/java/io/enmasse/systemtest/SysytemTestsErrorCollector.java b/systemtests/src/main/java/io/enmasse/systemtest/SysytemTestsErrorCollector.java
index c10c14460..b08a19f3e 100644
--- a/systemtests/src/main/java/io/enmasse/systemtest/SysytemTestsErrorCollector.java
+++ b/systemtests/src/main/java/io/enmasse/systemtest/SysytemTestsErrorCollector.java
@@ -4,7 +4,6 @@
  */
 package io.enmasse.systemtest;
 
-import io.enmasse.systemtest.CustomLogger;
 import org.hamcrest.Matcher;
 import org.slf4j.Logger;
 
diff --git a/systemtests/src/main/java/io/enmasse/systemtest/TestUtils.java b/systemtests/src/main/java/io/enmasse/systemtest/TestUtils.java
index a20d800ea..aaec7be44 100644
--- a/systemtests/src/main/java/io/enmasse/systemtest/TestUtils.java
+++ b/systemtests/src/main/java/io/enmasse/systemtest/TestUtils.java
@@ -41,25 +41,6 @@ public class TestUtils {
     private static Logger log = CustomLogger.getLogger();
     private static Set<String> validDestinationTypes = new HashSet<>(Arrays.asList("queue", "topic", "anycast", "multicast", "subscription"));
 
-    /**
-     * scale up/down specific Destination (type: StatefulSet) in address space
-     */
-    public static void setReplicas(Kubernetes kubernetes, AddressSpace addressSpace, Destination destination, int numReplicas, TimeoutBudget budget, long checkInterval) throws InterruptedException {
-        kubernetes.setStatefulSetReplicas(destination.getDeployment(), numReplicas);
-        waitForNBrokerReplicas(kubernetes, addressSpace.getNamespace(), numReplicas, true, destination, budget, checkInterval);
-    }
-
-    /**
-     * scale up/down specific Destination (type: StatefulSet) in address space WITHOUT WAIT
-     */
-    public static void setReplicas(Kubernetes kubernetes, AddressSpace addressSpace, Destination destination, int numReplicas) throws InterruptedException {
-        kubernetes.setStatefulSetReplicas(destination.getDeployment(), numReplicas);
-    }
-
-    public static void setReplicas(Kubernetes kubernetes, AddressSpace addressSpace, Destination destination, int numReplicas, TimeoutBudget budget) throws InterruptedException {
-        setReplicas(kubernetes, addressSpace, destination, numReplicas, budget, 5000);
-    }
-
     /**
      * scale up/down specific pod (type: Deployment) in address space
      */
@@ -84,13 +65,13 @@ public class TestUtils {
     /**
      * wait for expected count of Destination replicas in address space
      */
-    public static void waitForNBrokerReplicas(Kubernetes kubernetes, String infraUuid, int expectedReplicas, boolean readyRequired,
-                                              Destination destination, TimeoutBudget budget, long checkInterval) throws InterruptedException {
-        String statefulsetName = destination.getDeployment().startsWith("broker") ?
-                String.format("broker-pooled-%s", infraUuid) : destination.getDeployment();
+    public static void waitForNBrokerReplicas(AddressApiClient addressApiClient, Kubernetes kubernetes, AddressSpace addressSpace, int expectedReplicas, boolean readyRequired,
+                                              Destination destination, TimeoutBudget budget, long checkInterval) throws Exception {
+        Address address = getAddressObject(addressApiClient.getAddresses(addressSpace, Optional.of(destination.getName())));
+        String statefulsetName = (String) address.getAnnotations().get("cluster_id");
         Map<String, String> labels = new HashMap<>();
         labels.put("role", "broker");
-        labels.put("infraUuid", infraUuid);
+        labels.put("infraUuid", addressSpace.getInfraUuid());
         waitForNReplicas(kubernetes,
                 expectedReplicas,
                 readyRequired,
@@ -100,8 +81,8 @@ public class TestUtils {
                 checkInterval);
     }
 
-    public static void waitForNBrokerReplicas(Kubernetes kubernetes, String infraUuid, int expectedReplicas, Destination destination, TimeoutBudget budget) throws InterruptedException {
-        waitForNBrokerReplicas(kubernetes, infraUuid, expectedReplicas, true, destination, budget, 5000);
+    public static void waitForNBrokerReplicas(AddressApiClient addressApiClient, Kubernetes kubernetes, AddressSpace addressSpace, int expectedReplicas, Destination destination, TimeoutBudget budget) throws Exception {
+        waitForNBrokerReplicas(addressApiClient, kubernetes, addressSpace, expectedReplicas, true, destination, budget, 5000);
     }
 
 
@@ -706,6 +687,11 @@ public class TestUtils {
         String name = metadata.getString("name");
         String uid = metadata.getString("uid");
         String addressSpaceName = metadata.getString("addressSpace");
+        JsonObject annotationsJson = metadata.getJsonObject("annotations");
+        Map<String, Object> annotations = new HashMap<>();
+        if (annotationsJson != null) {
+            annotations = annotationsJson.getMap();
+        }
 
         JsonObject status = addressJsonObject.getJsonObject("status");
         boolean isReady = status.getBoolean("isReady");
@@ -718,7 +704,7 @@ public class TestUtils {
             }
         } catch (Exception ignored) {
         }
-        return new Address(addressSpaceName, address, name, type, plan, phase, isReady, messages, uid);
+        return new Address(addressSpaceName, address, annotations, name, type, plan, phase, isReady, messages, uid);
     }
 
     /**
@@ -946,7 +932,28 @@ public class TestUtils {
      * @param fn    request function
      * @return
      */
-    public static <T> Boolean doCommandNTimes(int retry, Callable<Boolean> fn) throws Exception {
+    public static <T> T runUntilPass(int retry, Callable<T> fn) throws InterruptedException {
+        for (int i = 0; i < retry; i++) {
+            try {
+                log.info("Running command, attempt: {}", i);
+                return fn.call();
+            } catch (Exception ex) {
+                log.info("Command failed");
+                ex.printStackTrace();
+            }
+            Thread.sleep(1000);
+        }
+        throw new IllegalStateException(String.format("Command wasn't pass in %s attempts", retry));
+    }
+
+    /**
+     * Repeat command n-times
+     *
+     * @param retry count of remaining retries
+     * @param fn    request function
+     * @return
+     */
+    public static Boolean doCommandNTimes(int retry, Callable<Boolean> fn) throws Exception {
         if (retry == 0)
             throw new IllegalStateException(String.format("Operation was not correctly completed within %d attempts", retry));
         Boolean result = fn.call();
diff --git a/systemtests/src/main/java/io/enmasse/systemtest/amqp/AmqpClient.java b/systemtests/src/main/java/io/enmasse/systemtest/amqp/AmqpClient.java
index 6b0bffc58..c2d6b772e 100644
--- a/systemtests/src/main/java/io/enmasse/systemtest/amqp/AmqpClient.java
+++ b/systemtests/src/main/java/io/enmasse/systemtest/amqp/AmqpClient.java
@@ -17,7 +17,9 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Optional;
-import java.util.concurrent.*;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
 
diff --git a/systemtests/src/main/java/io/enmasse/systemtest/amqp/ReceiverStatus.java b/systemtests/src/main/java/io/enmasse/systemtest/amqp/ReceiverStatus.java
index 2c4de9d19..4c1f69a76 100644
--- a/systemtests/src/main/java/io/enmasse/systemtest/amqp/ReceiverStatus.java
+++ b/systemtests/src/main/java/io/enmasse/systemtest/amqp/ReceiverStatus.java
@@ -11,5 +11,6 @@ import java.util.concurrent.Future;
 
 public interface ReceiverStatus {
     Future<List<Message>> getResult();
+
     int getNumReceived();
 }
diff --git a/systemtests/src/main/java/io/enmasse/systemtest/apiclients/AddressApiClient.java b/systemtests/src/main/java/io/enmasse/systemtest/apiclients/AddressApiClient.java
index d7d6a5bbe..d82aa5ee7 100644
--- a/systemtests/src/main/java/io/enmasse/systemtest/apiclients/AddressApiClient.java
+++ b/systemtests/src/main/java/io/enmasse/systemtest/apiclients/AddressApiClient.java
@@ -342,7 +342,7 @@ public class AddressApiClient extends ApiClient {
             int start = 0;
             try {
                 while (start < destinations.length) {
-                    Destination[] splice = Arrays.copyOfRange(destinations, start,  Math.min(start + batchSize, destinations.length));
+                    Destination[] splice = Arrays.copyOfRange(destinations, start, Math.min(start + batchSize, destinations.length));
                     JsonObject payload = createAddressListPayloadJson(addressSpace, splice);
                     createAddresses(addressSpace, payload, HTTP_CREATED);
                     start += splice.length;
@@ -392,7 +392,7 @@ public class AddressApiClient extends ApiClient {
         List<Destination> current = new ArrayList<>(TestUtils.convertToListAddress(response, Destination.class, object -> true));
 
         List<Destination> toCreate = new ArrayList<>(Arrays.asList(destinations));
-        List<Destination> toDelete= new ArrayList<>(current);
+        List<Destination> toDelete = new ArrayList<>(current);
 
         toDelete.removeAll(toCreate);
         toCreate.removeAll(current);
diff --git a/systemtests/src/main/java/io/enmasse/systemtest/apiclients/ApiClient.java b/systemtests/src/main/java/io/enmasse/systemtest/apiclients/ApiClient.java
index 4479ab4e7..93d765420 100644
--- a/systemtests/src/main/java/io/enmasse/systemtest/apiclients/ApiClient.java
+++ b/systemtests/src/main/java/io/enmasse/systemtest/apiclients/ApiClient.java
@@ -10,7 +10,6 @@ import io.fabric8.zjsonpatch.internal.guava.Strings;
 import io.vertx.core.AsyncResult;
 import io.vertx.core.Vertx;
 import io.vertx.core.json.JsonObject;
-import io.vertx.ext.web.client.HttpRequest;
 import io.vertx.ext.web.client.HttpResponse;
 import io.vertx.ext.web.client.WebClient;
 import io.vertx.ext.web.client.WebClientOptions;
@@ -123,17 +122,17 @@ public abstract class ApiClient {
         CompletableFuture<JsonObject> responsePromise = new CompletableFuture<>();
 
         return doRequestNTimes(initRetry, () -> {
-                client.get(endpoint.getPort(), endpoint.getHost(), path)
-                        .as(BodyCodec.jsonObject())
-                        .putHeader(HttpHeaders.AUTHORIZATION, authzString)
-                        .send(ar -> responseHandler(ar,
-                                responsePromise,
-                                expectedCode,
-                                String.format("Error: get %s %s", type, name)));
-                return responsePromise.get(30, TimeUnit.SECONDS);
-            },
-            Optional.of(endpointSupplier),
-            Optional.empty());
+                    client.get(endpoint.getPort(), endpoint.getHost(), path)
+                            .as(BodyCodec.jsonObject())
+                            .putHeader(HttpHeaders.AUTHORIZATION, authzString)
+                            .send(ar -> responseHandler(ar,
+                                    responsePromise,
+                                    expectedCode,
+                                    String.format("Error: get %s %s", type, name)));
+                    return responsePromise.get(30, TimeUnit.SECONDS);
+                },
+                Optional.of(endpointSupplier),
+                Optional.empty());
     }
 
     protected void createResource(String type, String basePath, JsonObject data) throws Exception {
@@ -147,7 +146,7 @@ public abstract class ApiClient {
         doRequestNTimes(initRetry, () -> {
                     client.post(endpoint.getPort(), endpoint.getHost(), basePath)
                             .timeout(20_000)
-                            .putHeader(HttpHeaders.AUTHORIZATION.toString(), authzString)
+                            .putHeader(HttpHeaders.AUTHORIZATION, authzString)
                             .as(BodyCodec.jsonObject())
                             .sendJsonObject(data, ar -> {
                                 responseHandler(ar,
@@ -174,7 +173,7 @@ public abstract class ApiClient {
         doRequestNTimes(initRetry, () -> {
                     client.put(endpoint.getPort(), endpoint.getHost(), path)
                             .timeout(20_000)
-                            .putHeader(HttpHeaders.AUTHORIZATION.toString(), authzString)
+                            .putHeader(HttpHeaders.AUTHORIZATION, authzString)
                             .as(BodyCodec.jsonObject())
                             .sendJsonObject(data, ar -> responseHandler(ar,
                                     responsePromise,
@@ -197,7 +196,7 @@ public abstract class ApiClient {
         doRequestNTimes(initRetry, () -> {
                     client.delete(endpoint.getPort(), endpoint.getHost(), path)
                             .as(BodyCodec.jsonObject())
-                            .putHeader(HttpHeaders.AUTHORIZATION.toString(), authzString)
+                            .putHeader(HttpHeaders.AUTHORIZATION, authzString)
                             .timeout(20_000)
                             .send(ar -> responseHandler(ar,
                                     responsePromise,
diff --git a/systemtests/src/main/java/io/enmasse/systemtest/mqtt/MqttClientFactory.java b/systemtests/src/main/java/io/enmasse/systemtest/mqtt/MqttClientFactory.java
index 00ce183ac..0a8f34de1 100644
--- a/systemtests/src/main/java/io/enmasse/systemtest/mqtt/MqttClientFactory.java
+++ b/systemtests/src/main/java/io/enmasse/systemtest/mqtt/MqttClientFactory.java
@@ -6,17 +6,7 @@
 package io.enmasse.systemtest.mqtt;
 
 import io.enmasse.systemtest.*;
-import org.eclipse.paho.client.mqttv3.IMqttClient;
-import org.eclipse.paho.client.mqttv3.IMqttDeliveryToken;
-import org.eclipse.paho.client.mqttv3.IMqttMessageListener;
-import org.eclipse.paho.client.mqttv3.IMqttToken;
-import org.eclipse.paho.client.mqttv3.MqttCallback;
-import org.eclipse.paho.client.mqttv3.MqttClient;
-import org.eclipse.paho.client.mqttv3.MqttConnectOptions;
-import org.eclipse.paho.client.mqttv3.MqttException;
-import org.eclipse.paho.client.mqttv3.MqttMessage;
-import org.eclipse.paho.client.mqttv3.MqttPersistenceException;
-import org.eclipse.paho.client.mqttv3.MqttTopic;
+import org.eclipse.paho.client.mqttv3.*;
 import org.eclipse.paho.client.mqttv3.persist.MemoryPersistence;
 import org.slf4j.Logger;
 
@@ -28,13 +18,7 @@ import java.net.Socket;
 import java.security.NoSuchAlgorithmException;
 import java.security.SecureRandom;
 import java.security.cert.X509Certificate;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-import java.util.UUID;
+import java.util.*;
 
 public class MqttClientFactory {
 
@@ -71,8 +55,7 @@ public class MqttClientFactory {
                 Arrays.toString(protocols)));
     }
 
-    public Builder build()
-    {
+    public Builder build() {
         return new Builder() {
             AddressSpace addressSpace = defaultAddressSpace;
             MqttConnectOptions mqttConnectOptions = new MqttConnectOptions();
@@ -386,12 +369,12 @@ public class MqttClientFactory {
         }
 
         @Override
-        public void publish(String topic, byte[] payload, int qos, boolean retained) throws MqttException, MqttPersistenceException {
+        public void publish(String topic, byte[] payload, int qos, boolean retained) throws MqttException {
             this.mqttClient.publish(topic, payload, qos, retained);
         }
 
         @Override
-        public void publish(String topic, MqttMessage message) throws MqttException, MqttPersistenceException {
+        public void publish(String topic, MqttMessage message) throws MqttException {
             this.mqttClient.publish(topic, message);
         }
 
@@ -448,8 +431,11 @@ public class MqttClientFactory {
     public interface Builder {
 
         Builder addressSpace(AddressSpace addressSpace);
+
         Builder mqttConnectionOptions(MqttConnectOptions options);
+
         Builder clientId(String clientId);
+
         IMqttClient create() throws Exception;
     }
 }
diff --git a/systemtests/src/main/java/io/enmasse/systemtest/mqtt/MqttUtils.java b/systemtests/src/main/java/io/enmasse/systemtest/mqtt/MqttUtils.java
index e1d2b621f..5b99cfa47 100644
--- a/systemtests/src/main/java/io/enmasse/systemtest/mqtt/MqttUtils.java
+++ b/systemtests/src/main/java/io/enmasse/systemtest/mqtt/MqttUtils.java
@@ -25,9 +25,9 @@ public class MqttUtils {
         CompletableFuture<Void> future = CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]));
         try {
             future.get(timeout, timeUnit);
-            return (int)futures.stream().filter(CompletableFuture::isDone).count();
+            return (int) futures.stream().filter(CompletableFuture::isDone).count();
         } catch (TimeoutException e) {
-            return (int)futures.stream().filter(CompletableFuture::isDone).count();
+            return (int) futures.stream().filter(CompletableFuture::isDone).count();
         }
     }
 
diff --git a/systemtests/src/main/java/io/enmasse/systemtest/resources/AddressSpacePlan.java b/systemtests/src/main/java/io/enmasse/systemtest/resources/AddressSpacePlan.java
index f463806e3..abb345655 100644
--- a/systemtests/src/main/java/io/enmasse/systemtest/resources/AddressSpacePlan.java
+++ b/systemtests/src/main/java/io/enmasse/systemtest/resources/AddressSpacePlan.java
@@ -5,7 +5,6 @@
 package io.enmasse.systemtest.resources;
 
 import io.enmasse.systemtest.AddressSpaceType;
-import io.enmasse.systemtest.AddressType;
 import io.enmasse.systemtest.apiclients.AdminApiClient;
 import io.vertx.core.json.JsonArray;
 import io.vertx.core.json.JsonObject;
@@ -40,6 +39,7 @@ public class AddressSpacePlan {
     public String getName() {
         return name;
     }
+
     public String getInfraConfigName() {
         return infraConfigName;
     }
diff --git a/systemtests/src/main/java/io/enmasse/systemtest/resources/CliOutputData.java b/systemtests/src/main/java/io/enmasse/systemtest/resources/CliOutputData.java
index 0b8815073..bf43a4959 100644
--- a/systemtests/src/main/java/io/enmasse/systemtest/resources/CliOutputData.java
+++ b/systemtests/src/main/java/io/enmasse/systemtest/resources/CliOutputData.java
@@ -102,6 +102,7 @@ public class CliOutputData {
             try {
                 this.status = parsedData[5];
             } catch (Exception ignored) {
+                this.status = "";
             }
         }
 
@@ -139,6 +140,7 @@ public class CliOutputData {
             try {
                 this.status = parsedData[8];
             } catch (Exception ignored) {
+                this.status = "";
             }
         }
 
diff --git a/systemtests/src/main/java/io/enmasse/systemtest/selenium/ISeleniumProvider.java b/systemtests/src/main/java/io/enmasse/systemtest/selenium/ISeleniumProvider.java
index cc448873a..adc507675 100644
--- a/systemtests/src/main/java/io/enmasse/systemtest/selenium/ISeleniumProvider.java
+++ b/systemtests/src/main/java/io/enmasse/systemtest/selenium/ISeleniumProvider.java
@@ -16,6 +16,7 @@ public interface ISeleniumProvider {
     SeleniumProvider selenium = new SeleniumProvider();
 
     WebDriver buildDriver() throws Exception;
+
     void restartSeleniumContainer() throws Exception;
 
     @AfterEach
diff --git a/systemtests/src/main/java/io/enmasse/systemtest/selenium/ISeleniumProviderChrome.java b/systemtests/src/main/java/io/enmasse/systemtest/selenium/ISeleniumProviderChrome.java
index e247137ef..6824ee13a 100644
--- a/systemtests/src/main/java/io/enmasse/systemtest/selenium/ISeleniumProviderChrome.java
+++ b/systemtests/src/main/java/io/enmasse/systemtest/selenium/ISeleniumProviderChrome.java
@@ -6,7 +6,6 @@ package io.enmasse.systemtest.selenium;
 
 import io.enmasse.systemtest.TestUtils;
 import org.junit.jupiter.api.AfterAll;
-import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeAll;
 import org.openqa.selenium.WebDriver;
 
diff --git a/systemtests/src/main/java/io/enmasse/systemtest/selenium/ISeleniumProviderFirefox.java b/systemtests/src/main/java/io/enmasse/systemtest/selenium/ISeleniumProviderFirefox.java
index 2c0a90364..015ca5782 100644
--- a/systemtests/src/main/java/io/enmasse/systemtest/selenium/ISeleniumProviderFirefox.java
+++ b/systemtests/src/main/java/io/enmasse/systemtest/selenium/ISeleniumProviderFirefox.java
@@ -6,7 +6,6 @@ package io.enmasse.systemtest.selenium;
 
 import io.enmasse.systemtest.TestUtils;
 import org.junit.jupiter.api.AfterAll;
-import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeAll;
 import org.openqa.selenium.WebDriver;
 
diff --git a/systemtests/src/main/java/io/enmasse/systemtest/selenium/page/ConsoleWebPage.java b/systemtests/src/main/java/io/enmasse/systemtest/selenium/page/ConsoleWebPage.java
index b6067c96d..a6c0e4c78 100644
--- a/systemtests/src/main/java/io/enmasse/systemtest/selenium/page/ConsoleWebPage.java
+++ b/systemtests/src/main/java/io/enmasse/systemtest/selenium/page/ConsoleWebPage.java
@@ -8,7 +8,6 @@ package io.enmasse.systemtest.selenium.page;
 import com.paulhammant.ngwebdriver.ByAngular;
 import io.enmasse.systemtest.*;
 import io.enmasse.systemtest.apiclients.AddressApiClient;
-import io.enmasse.systemtest.selenium.page.OpenshiftWebPage;
 import io.enmasse.systemtest.selenium.SeleniumProvider;
 import io.enmasse.systemtest.selenium.resources.*;
 import org.openqa.selenium.By;
@@ -19,7 +18,6 @@ import org.slf4j.Logger;
 
 import java.time.Duration;
 import java.util.ArrayList;
-import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
@@ -307,7 +305,7 @@ public class ConsoleWebPage implements IWebPage {
     public List<ConnectionWebItem> getConnectionItems(int expectedCount) {
         List<ConnectionWebItem> connectionItems = new ArrayList<>();
         long endTime = System.currentTimeMillis() + 30000;
-        while(connectionItems.size() != expectedCount && endTime > System.currentTimeMillis()) {
+        while (connectionItems.size() != expectedCount && endTime > System.currentTimeMillis()) {
             log.info("First iteration waiting for {} connections items", expectedCount);
             WebElement content = getContentContainer();
             List<WebElement> elements = content.findElements(By.className("list-group-item"));
diff --git a/systemtests/src/main/java/io/enmasse/systemtest/selenium/resources/ToolbarType.java b/systemtests/src/main/java/io/enmasse/systemtest/selenium/resources/ToolbarType.java
index ca9b486f8..70e3522a6 100644
--- a/systemtests/src/main/java/io/enmasse/systemtest/selenium/resources/ToolbarType.java
+++ b/systemtests/src/main/java/io/enmasse/systemtest/selenium/resources/ToolbarType.java
@@ -14,5 +14,5 @@ public enum ToolbarType {
         public String toString() {
             return "connectionToolbar";
         }
-    };
+    }
 }
diff --git a/systemtests/src/main/resources/rhea.html b/systemtests/src/main/resources/rhea.html
index 8084b591c..1bde5f920 100644
--- a/systemtests/src/main/resources/rhea.html
+++ b/systemtests/src/main/resources/rhea.html
@@ -2,14 +2,14 @@
 <html>
 <head>
     <title>cli-rhea client</title>
-    <meta http-equiv="content-type" content="text/html;charset=utf-8" />
+    <meta http-equiv="content-type" content="text/html;charset=utf-8"/>
     <script type="text/javascript" src="./rhea.js"></script>
 </head>
 
 <body>
-    <h1>Browser based testing</h1>
+<h1>Browser based testing</h1>
 
-    <script type="text/javascript">
+<script type="text/javascript">
         var connect_to_enmasse = function(server, address, count, username, password, protocols){
             var requests = parseInt(count);
             var current = 1;
@@ -43,7 +43,8 @@
             connection.open_receiver(address);
             sender = connection.open_sender(address);
         }
-    </script>
+
+</script>
 
 </body>
 </html>
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/bases/TestBase.java b/systemtests/src/test/java/io/enmasse/systemtest/bases/TestBase.java
index bd864c920..90dc107af 100644
--- a/systemtests/src/test/java/io/enmasse/systemtest/bases/TestBase.java
+++ b/systemtests/src/test/java/io/enmasse/systemtest/bases/TestBase.java
@@ -335,22 +335,6 @@ public abstract class TestBase implements ITestBase, ITestSeparator {
         return TestUtils.getDestinationsObjects(addressApiClient, addressSpace, addressName, new ArrayList<>());
     }
 
-    /**
-     * scale up/down destination (StatefulSet) to count of replicas, includes waiting for expected replicas
-     */
-    private void scale(AddressSpace addressSpace, Destination destination, int numReplicas, long checkInterval) throws Exception {
-        TimeoutBudget budget = new TimeoutBudget(5, TimeUnit.MINUTES);
-        TestUtils.setReplicas(kubernetes, addressSpace, destination, numReplicas, budget, checkInterval);
-    }
-
-    private void scaleWithoutWait(AddressSpace addressSpace, Destination destination, int numReplicas) throws Exception {
-        TestUtils.setReplicas(kubernetes, addressSpace, destination, numReplicas);
-    }
-
-    void scale(AddressSpace addressSpace, Destination destination, int numReplicas) throws Exception {
-        scale(addressSpace, destination, numReplicas, 5000);
-    }
-
     protected void scaleKeycloak(int numReplicas) throws Exception {
         scaleInGlobal("keycloak", numReplicas);
     }
@@ -593,11 +577,7 @@ public abstract class TestBase implements ITestBase, ITestSeparator {
     }
 
     private void waitForBrokerReplicas(AddressSpace addressSpace, Destination destination, int expectedReplicas, boolean readyRequired, TimeoutBudget budget, long checkInterval) throws Exception {
-        TestUtils.waitForNBrokerReplicas(kubernetes, addressSpace.getInfraUuid(), expectedReplicas, readyRequired, destination, budget, checkInterval);
-    }
-
-    protected void waitForBrokerReplicas(AddressSpace addressSpace, Destination destination, int expectedReplicas, boolean readyRequired, TimeoutBudget budget) throws Exception {
-        waitForBrokerReplicas(addressSpace, destination, expectedReplicas, readyRequired, budget, 5000);
+        TestUtils.waitForNBrokerReplicas(addressApiClient, kubernetes, addressSpace, expectedReplicas, readyRequired, destination, budget, checkInterval);
     }
 
     private void waitForBrokerReplicas(AddressSpace addressSpace, Destination destination,
@@ -607,7 +587,7 @@ public abstract class TestBase implements ITestBase, ITestSeparator {
 
     protected void waitForBrokerReplicas(AddressSpace addressSpace, Destination destination, int expectedReplicas) throws
             Exception {
-        TimeoutBudget budget = new TimeoutBudget(1, TimeUnit.MINUTES);
+        TimeoutBudget budget = new TimeoutBudget(5, TimeUnit.MINUTES);
         waitForBrokerReplicas(addressSpace, destination, expectedReplicas, budget);
     }
 
@@ -620,21 +600,6 @@ public abstract class TestBase implements ITestBase, ITestSeparator {
         TestUtils.waitForNReplicas(kubernetes, expectedReplicas, labels, budget);
     }
 
-    protected void waitForAutoScale(AddressSpace addressSpace, Destination dest, int setValue, int expectedValue) throws Exception {
-        log.info("Set '{}' replicas and wait for autoscale to '{}'", setValue, expectedValue);
-        CompletableFuture<Void> scaleCheckerDown = CompletableFuture.runAsync(() -> {
-            try {
-                waitForBrokerReplicas(addressSpace, dest, setValue, false, new TimeoutBudget(3, TimeUnit.MINUTES), 1);
-                log.info("Waiting for expected replicas {} finished!", setValue);
-            } catch (Exception e) {
-                e.printStackTrace();
-            }
-        });
-        scaleWithoutWait(addressSpace, dest, setValue);
-        scaleCheckerDown.get(3, TimeUnit.MINUTES);
-        waitForBrokerReplicas(addressSpace, dest, expectedValue, new TimeoutBudget(2, TimeUnit.MINUTES));
-    }
-
     /**
      * Wait for destinations are in isReady=true state within default timeout (5 MINUTE)
      */
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/bases/TestBaseWithShared.java b/systemtests/src/test/java/io/enmasse/systemtest/bases/TestBaseWithShared.java
index 023ae677b..5aea5c42c 100644
--- a/systemtests/src/test/java/io/enmasse/systemtest/bases/TestBaseWithShared.java
+++ b/systemtests/src/test/java/io/enmasse/systemtest/bases/TestBaseWithShared.java
@@ -95,10 +95,6 @@ public abstract class TestBaseWithShared extends TestBase {
         super.createAddressSpace(addressSpace);
     }
 
-    protected void scale(Destination destination, int numReplicas) throws Exception {
-        scale(sharedAddressSpace, destination, numReplicas);
-    }
-
     /**
      * get all addresses except 'dummy-address'
      */
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/bases/web/WebConsolePlansTest.java b/systemtests/src/test/java/io/enmasse/systemtest/bases/web/WebConsolePlansTest.java
index 1d1735398..29c7e4c5e 100644
--- a/systemtests/src/test/java/io/enmasse/systemtest/bases/web/WebConsolePlansTest.java
+++ b/systemtests/src/test/java/io/enmasse/systemtest/bases/web/WebConsolePlansTest.java
@@ -13,8 +13,8 @@ import io.enmasse.systemtest.resources.AddressPlan;
 import io.enmasse.systemtest.resources.AddressResource;
 import io.enmasse.systemtest.resources.AddressSpacePlan;
 import io.enmasse.systemtest.resources.AddressSpaceResource;
-import io.enmasse.systemtest.selenium.page.ConsoleWebPage;
 import io.enmasse.systemtest.selenium.ISeleniumProvider;
+import io.enmasse.systemtest.selenium.page.ConsoleWebPage;
 import io.enmasse.systemtest.standard.QueueTest;
 import io.enmasse.systemtest.standard.TopicTest;
 import org.junit.jupiter.api.AfterEach;
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/bases/web/WebSocketBrowserTest.java b/systemtests/src/test/java/io/enmasse/systemtest/bases/web/WebSocketBrowserTest.java
index ad9f499af..5153cbbca 100644
--- a/systemtests/src/test/java/io/enmasse/systemtest/bases/web/WebSocketBrowserTest.java
+++ b/systemtests/src/test/java/io/enmasse/systemtest/bases/web/WebSocketBrowserTest.java
@@ -9,7 +9,6 @@ import io.enmasse.systemtest.*;
 import io.enmasse.systemtest.bases.TestBaseWithShared;
 import io.enmasse.systemtest.selenium.ISeleniumProvider;
 import io.enmasse.systemtest.selenium.page.RheaWebPage;
-import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeEach;
 import org.slf4j.Logger;
 
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/brokered/SmokeTest.java b/systemtests/src/test/java/io/enmasse/systemtest/brokered/SmokeTest.java
index afa5f9e60..f3a6ea008 100644
--- a/systemtests/src/test/java/io/enmasse/systemtest/brokered/SmokeTest.java
+++ b/systemtests/src/test/java/io/enmasse/systemtest/brokered/SmokeTest.java
@@ -5,8 +5,8 @@
 package io.enmasse.systemtest.brokered;
 
 import io.enmasse.systemtest.*;
-import io.enmasse.systemtest.amqp.AmqpClient;
 import io.enmasse.systemtest.ability.ITestBaseBrokered;
+import io.enmasse.systemtest.amqp.AmqpClient;
 import io.enmasse.systemtest.bases.TestBaseWithShared;
 import io.enmasse.systemtest.standard.QueueTest;
 import org.apache.qpid.proton.message.Message;
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/brokered/TopicTest.java b/systemtests/src/test/java/io/enmasse/systemtest/brokered/TopicTest.java
index 85d3551f9..15ee7651a 100644
--- a/systemtests/src/test/java/io/enmasse/systemtest/brokered/TopicTest.java
+++ b/systemtests/src/test/java/io/enmasse/systemtest/brokered/TopicTest.java
@@ -4,8 +4,8 @@
  */
 package io.enmasse.systemtest.brokered;
 
-import io.enmasse.systemtest.*;
 import io.enmasse.systemtest.Destination;
+import io.enmasse.systemtest.*;
 import io.enmasse.systemtest.ability.ITestBaseBrokered;
 import io.enmasse.systemtest.amqp.AmqpClient;
 import io.enmasse.systemtest.bases.TestBaseWithShared;
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/brokered/authz/AuthorizationTest.java b/systemtests/src/test/java/io/enmasse/systemtest/brokered/authz/AuthorizationTest.java
index 1cb82db68..80ed45307 100644
--- a/systemtests/src/test/java/io/enmasse/systemtest/brokered/authz/AuthorizationTest.java
+++ b/systemtests/src/test/java/io/enmasse/systemtest/brokered/authz/AuthorizationTest.java
@@ -6,12 +6,8 @@ package io.enmasse.systemtest.brokered.authz;
 
 import io.enmasse.systemtest.ability.ITestBaseBrokered;
 import io.enmasse.systemtest.bases.authz.AuthorizationTestBase;
-import org.junit.jupiter.api.Disabled;
-import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 
-import static io.enmasse.systemtest.TestTag.nonPR;
-
 class AuthorizationTest extends AuthorizationTestBase implements ITestBaseBrokered {
 
     @Test
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/brokered/clients/openwire/MsgPatternsTest.java b/systemtests/src/test/java/io/enmasse/systemtest/brokered/clients/openwire/MsgPatternsTest.java
index 00dfb8c58..89dda5813 100644
--- a/systemtests/src/test/java/io/enmasse/systemtest/brokered/clients/openwire/MsgPatternsTest.java
+++ b/systemtests/src/test/java/io/enmasse/systemtest/brokered/clients/openwire/MsgPatternsTest.java
@@ -10,7 +10,6 @@ import io.enmasse.systemtest.bases.clients.ClientTestBase;
 import io.enmasse.systemtest.messagingclients.openwire.OpenwireJMSClientReceiver;
 import io.enmasse.systemtest.messagingclients.openwire.OpenwireJMSClientSender;
 import io.enmasse.systemtest.resolvers.ArtemisManagementParameterResolver;
-import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/brokered/clients/proton/java/MsgPatternsInternalTest.java b/systemtests/src/test/java/io/enmasse/systemtest/brokered/clients/proton/java/MsgPatternsInternalTest.java
index e87a2ce49..9c4650db4 100644
--- a/systemtests/src/test/java/io/enmasse/systemtest/brokered/clients/proton/java/MsgPatternsInternalTest.java
+++ b/systemtests/src/test/java/io/enmasse/systemtest/brokered/clients/proton/java/MsgPatternsInternalTest.java
@@ -8,7 +8,6 @@ import io.enmasse.systemtest.ability.ITestBaseBrokered;
 import io.enmasse.systemtest.bases.clients.ClusterClientTestBase;
 import io.enmasse.systemtest.messagingclients.proton.java.ProtonJMSClientReceiver;
 import io.enmasse.systemtest.messagingclients.proton.java.ProtonJMSClientSender;
-import io.enmasse.systemtest.messagingclients.proton.python.PythonClientReceiver;
 import org.junit.jupiter.api.Test;
 
 class MsgPatternsInternalTest extends ClusterClientTestBase implements ITestBaseBrokered {
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/brokered/web/FirefoxWebConsoleTest.java b/systemtests/src/test/java/io/enmasse/systemtest/brokered/web/FirefoxWebConsoleTest.java
index 75c4878f5..fa612f5b6 100644
--- a/systemtests/src/test/java/io/enmasse/systemtest/brokered/web/FirefoxWebConsoleTest.java
+++ b/systemtests/src/test/java/io/enmasse/systemtest/brokered/web/FirefoxWebConsoleTest.java
@@ -160,7 +160,8 @@ class FirefoxWebConsoleTest extends WebConsoleTest implements ITestBaseBrokered,
     }
 
     @Test
-    @Disabled("disabled while sdavey changes it with changes to regex in addr names") //TODO(sdavey)
+    @Disabled("disabled while sdavey changes it with changes to regex in addr names")
+        //TODO(sdavey)
     void testCreateAddressWithSymbolsAt61stCharIndex() throws Exception {
         doTestCreateAddressWithSymbolsAt61stCharIndex(
                 Destination.queue("queue10charHere-10charHere-10charHere-10charHere-10charHere-1",
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/common/CommonTest.java b/systemtests/src/test/java/io/enmasse/systemtest/common/CommonTest.java
index 81e3a33ee..7a390dfce 100644
--- a/systemtests/src/test/java/io/enmasse/systemtest/common/CommonTest.java
+++ b/systemtests/src/test/java/io/enmasse/systemtest/common/CommonTest.java
@@ -7,6 +7,7 @@ package io.enmasse.systemtest.common;
 import io.enmasse.systemtest.*;
 import io.enmasse.systemtest.bases.TestBase;
 import io.enmasse.systemtest.cmdclients.CRDCmdClient;
+import io.fabric8.kubernetes.api.model.Pod;
 import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
@@ -64,8 +65,12 @@ class CommonTest extends TestBase {
         assertCanConnect(brokered, user, brokeredAddresses);
         assertCanConnect(standard, user, standardAddresses);
 
-        //number of pods running before restarting any
-        int runningPodsBefore = kubernetes.listPods().size();
+        log.info("------------------------------------------------------------");
+        log.info("------------------- Start with restating -------------------");
+        log.info("------------------------------------------------------------");
+
+        List<Pod> pods = kubernetes.listPods();
+        int runningPodsBefore = pods.size();
         log.info("Number of running pods before restarting any: {}", runningPodsBefore);
 
         for (Label label : labels) {
@@ -78,11 +83,17 @@ class CommonTest extends TestBase {
 
         log.info("Restarting whole enmasse");
         CRDCmdClient.deletePodByLabel("app", kubernetes.getEnmasseAppLabel());
-        Thread.sleep(120_000);
+        Thread.sleep(180_000);
         TestUtils.waitForExpectedReadyPods(kubernetes, runningPodsBefore, new TimeoutBudget(120, TimeUnit.SECONDS));
         TestUtils.waitForDestinationsReady(addressApiClient, standard, new TimeoutBudget(180, TimeUnit.SECONDS),
                 standardAddresses.toArray(new Destination[0]));
         assertSystemWorks(brokered, standard, user, brokeredAddresses, standardAddresses);
+
+        //TODO: Uncomment when #2127 will be fixedy
+
+//        Pod qdrouter = pods.stream().filter(pod -> pod.getMetadata().getName().contains("qdrouter")).collect(Collectors.toList()).get(0);
+//        kubernetes.deletePod(environment.namespace(), qdrouter.getMetadata().getName());
+//        assertSystemWorks(brokered, standard, user, brokeredAddresses, standardAddresses);
     }
 
     @Test
@@ -102,12 +113,12 @@ class CommonTest extends TestBase {
     private void assertSystemWorks(AddressSpace brokered, AddressSpace standard, UserCredentials existingUser,
                                    List<Destination> brAddresses, List<Destination> stAddresses) throws Exception {
         log.info("Check if system works");
-        brokered = getAddressSpace(brokered.getName());
-        standard = getAddressSpace(standard.getName());
+        assertCanConnect(standard, existingUser, stAddresses);
+        assertCanConnect(brokered, existingUser, brAddresses);
+        getAddressSpace(brokered.getName());
+        getAddressSpace(standard.getName());
         createUser(brokered, new UserCredentials("jenda", "cenda"));
         createUser(standard, new UserCredentials("jura", "fura"));
-        assertCanConnect(brokered, existingUser, brAddresses);
-        assertCanConnect(standard, existingUser, stAddresses);
     }
 
     private class Label {
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/common/api/CustomResourceDefinitionAddressSpacesTest.java b/systemtests/src/test/java/io/enmasse/systemtest/common/api/CustomResourceDefinitionAddressSpacesTest.java
index e9c565b1f..665f0bb1b 100644
--- a/systemtests/src/test/java/io/enmasse/systemtest/common/api/CustomResourceDefinitionAddressSpacesTest.java
+++ b/systemtests/src/test/java/io/enmasse/systemtest/common/api/CustomResourceDefinitionAddressSpacesTest.java
@@ -21,6 +21,7 @@ import java.util.concurrent.TimeUnit;
 
 import static io.enmasse.systemtest.TestTag.isolated;
 import static io.enmasse.systemtest.cmdclients.CRDCmdClient.createCR;
+import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.junit.jupiter.api.Assertions.*;
@@ -115,8 +116,13 @@ public class CustomResourceDefinitionAddressSpacesTest extends TestBase {
             CliOutputData data = new CliOutputData(CRDCmdClient.getAddressSpace(namespace, Optional.of("wide")).getStdOut(),
                     CliOutputData.CliOutputDataType.ADDRESS_SPACE);
             assertTrue(((CliOutputData.AddressSpaceRow) data.getData(brokered.getName())).isReady());
-            assertTrue(((CliOutputData.AddressSpaceRow) data.getData(standard.getName()))
-                    .getStatus().contains("Following deployments and statefulsets are not ready"));
+            if(((CliOutputData.AddressSpaceRow) data.getData(brokered.getName())).isReady()) {
+                assertThat(((CliOutputData.AddressSpaceRow) data.getData(standard.getName())).getStatus(),
+                        containsString(""));
+            } else {
+                assertThat(((CliOutputData.AddressSpaceRow) data.getData(standard.getName())).getStatus(),
+                        containsString("Following deployments and statefulsets are not ready"));
+            }
 
             waitForAddressSpaceReady(standard, apiClient);
 
@@ -124,8 +130,8 @@ public class CustomResourceDefinitionAddressSpacesTest extends TestBase {
                     CliOutputData.CliOutputDataType.ADDRESS_SPACE);
             assertTrue(((CliOutputData.AddressSpaceRow) data.getData(brokered.getName())).isReady());
             assertTrue(((CliOutputData.AddressSpaceRow) data.getData(standard.getName())).isReady());
-            assertNull(((CliOutputData.AddressSpaceRow) data.getData(standard.getName()))
-                    .getStatus());
+            assertTrue(((CliOutputData.AddressSpaceRow) data.getData(standard.getName()))
+                    .getStatus().isEmpty());
 
 
             //===========================
@@ -188,7 +194,7 @@ public class CustomResourceDefinitionAddressSpacesTest extends TestBase {
                     DestinationPlan.STANDARD_LARGE_TOPIC.plan());
             assertEquals(((CliOutputData.AddressRow) data.getData(String.format("%s.%s", standard.getName(), anycast.getAddress()))).getPhase(),
                     "Configuring");
-            assertNotNull(((CliOutputData.AddressRow) data.getData(String.format("%s.%s", standard.getName(), topicStandard.getAddress()))).getStatus());
+            assertFalse(((CliOutputData.AddressRow) data.getData(String.format("%s.%s", standard.getName(), topicStandard.getAddress()))).getStatus().isEmpty());
 
             TestUtils.waitForDestinationsReady(apiClient, standard, new TimeoutBudget(5, TimeUnit.MINUTES), anycast, topicStandard);
 
@@ -200,7 +206,7 @@ public class CustomResourceDefinitionAddressSpacesTest extends TestBase {
                     DestinationPlan.STANDARD_LARGE_TOPIC.plan());
             assertEquals(((CliOutputData.AddressRow) data.getData(String.format("%s.%s", standard.getName(), anycast.getAddress()))).getPhase(),
                     "Active");
-            assertNull(((CliOutputData.AddressRow) data.getData(String.format("%s.%s", standard.getName(), topicStandard.getAddress()))).getStatus());
+            assertTrue(((CliOutputData.AddressRow) data.getData(String.format("%s.%s", standard.getName(), topicStandard.getAddress()))).getStatus().isEmpty());
 
             //===========================
             // Clean part
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/common/api/CustomResourceDefinitionAddressesTest.java b/systemtests/src/test/java/io/enmasse/systemtest/common/api/CustomResourceDefinitionAddressesTest.java
index bf8df5c75..e5f810360 100644
--- a/systemtests/src/test/java/io/enmasse/systemtest/common/api/CustomResourceDefinitionAddressesTest.java
+++ b/systemtests/src/test/java/io/enmasse/systemtest/common/api/CustomResourceDefinitionAddressesTest.java
@@ -133,7 +133,7 @@ public class CustomResourceDefinitionAddressesTest extends TestBase implements I
         List<String> dest2Expected = Arrays.asList(
                 String.format(addressString, "address", brokered.getName(), dest2.getName()),
                 String.format(addressString, "address.enmasse.io", brokered.getName(), dest2.getName()),
-        String.format(address2String, "address.enmasse.io", brokered.getName(), dest2.getName()));
+                String.format(address2String, "address.enmasse.io", brokered.getName(), dest2.getName()));
         assertTrue(dest2Expected.contains(output),
                 String.format("Unexpected response on create custom resource '%s': %s", address2, output));
         assertTrue(result.getRetCode(), String.format("Expected return code 0 on create custom resource '%s'", address2));
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/common/catalog/ServiceCatalogApiTest.java b/systemtests/src/test/java/io/enmasse/systemtest/common/catalog/ServiceCatalogApiTest.java
index a9779d741..3cf119d25 100644
--- a/systemtests/src/test/java/io/enmasse/systemtest/common/catalog/ServiceCatalogApiTest.java
+++ b/systemtests/src/test/java/io/enmasse/systemtest/common/catalog/ServiceCatalogApiTest.java
@@ -15,7 +15,6 @@ import org.junit.jupiter.api.*;
 import org.junit.jupiter.api.condition.DisabledIfEnvironmentVariable;
 import org.slf4j.Logger;
 
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Optional;
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/common/plans/PlansTest.java b/systemtests/src/test/java/io/enmasse/systemtest/common/plans/PlansTest.java
index d07de37bf..a371bc7ae 100644
--- a/systemtests/src/test/java/io/enmasse/systemtest/common/plans/PlansTest.java
+++ b/systemtests/src/test/java/io/enmasse/systemtest/common/plans/PlansTest.java
@@ -248,6 +248,53 @@ class PlansTest extends TestBase implements ISeleniumProviderChrome {
                 ), user);
     }
 
+    @Test
+    void testScalePooledBrokers() throws Exception {
+        //define and create address plans
+        List<AddressResource> addressResourcesQueue = Collections.singletonList(new AddressResource("broker", 0.99));
+        AddressPlan xlQueuePlan = new AddressPlan("pooled-xl-queue", AddressType.QUEUE, addressResourcesQueue);
+        plansProvider.createAddressPlan(xlQueuePlan);
+
+        //define and create address space plan
+        List<AddressSpaceResource> resources = Arrays.asList(
+                new AddressSpaceResource("broker", 10.0),
+                new AddressSpaceResource("router", 2.0),
+                new AddressSpaceResource("aggregate", 12.0));
+        List<AddressPlan> addressPlans = Collections.singletonList(xlQueuePlan);
+        AddressSpacePlan manyAddressesPlan = new AddressSpacePlan("many-brokers-plan",
+                "default", AddressSpaceType.STANDARD, resources, addressPlans);
+        plansProvider.createAddressSpacePlan(manyAddressesPlan);
+
+        //create address space plan with new plan
+        AddressSpace manyAddressesSpace = new AddressSpace("many-addresses-standard", AddressSpaceType.STANDARD,
+                manyAddressesPlan.getName(), AuthService.STANDARD);
+        createAddressSpace(manyAddressesSpace);
+
+        UserCredentials cred = new UserCredentials("testus", "papyrus");
+        createUser(manyAddressesSpace, cred);
+
+        ArrayList<Destination> dest = new ArrayList<>();
+        int destCount = 4;
+        int toDeleteCount = 2;
+        for (int i = 0; i < destCount; i++) {
+            dest.add(Destination.queue("xl-queue-" + i, xlQueuePlan.getName()));
+        }
+
+        setAddresses(manyAddressesSpace, dest.toArray(new Destination[0]));
+        for (Destination destination : dest) {
+            waitForBrokerReplicas(manyAddressesSpace, destination, 1);
+        }
+
+        assertCanConnect(manyAddressesSpace, cred, dest);
+
+        deleteAddresses(manyAddressesSpace, dest.subList(0, toDeleteCount).toArray(new Destination[0]));
+        for (Destination destination : dest.subList(toDeleteCount, destCount)) {
+            waitForBrokerReplicas(manyAddressesSpace, destination, 1);
+        }
+
+        assertCanConnect(manyAddressesSpace, cred, dest.subList(toDeleteCount, destCount));
+    }
+
     @Test
     @Tag(nonPR)
     @Disabled("test disabled as per-address limit enforcement has been removed")
@@ -343,7 +390,6 @@ class PlansTest extends TestBase implements ISeleniumProviderChrome {
 //    }
 
     @Test
-    @Disabled("test disabled due to issue: #1134")
     void testMessagePersistenceAfterAutoScale() throws Exception {
         //define and create address plans
         List<AddressResource> addressResourcesQueueAlpha = Collections.singletonList(new AddressResource("broker", 0.3));
@@ -399,8 +445,7 @@ class PlansTest extends TestBase implements ISeleniumProviderChrome {
 
         //remove addresses from first pod and wait for scale down
         deleteAddresses(messagePersistAddressSpace, queue1, queue2);
-        TestUtils.waitForNBrokerReplicas(kubernetes, messagePersistAddressSpace.getNamespace(), 1, queue4, new TimeoutBudget(2, TimeUnit.MINUTES));
-        //test failed in command above ^, functionality of test code below wasn't verified :) !TODO
+        TestUtils.waitForNBrokerReplicas(addressApiClient, kubernetes, messagePersistAddressSpace, 1, queue4, new TimeoutBudget(2, TimeUnit.MINUTES));
 
         //validate count of addresses
         Future<List<String>> addresses = getAddresses(messagePersistAddressSpace, Optional.empty());
@@ -447,7 +492,7 @@ class PlansTest extends TestBase implements ISeleniumProviderChrome {
         setAddresses(messagePersistAddressSpace, queue);
 
         //pod should have 2 replicas
-        TestUtils.waitForNBrokerReplicas(kubernetes, messagePersistAddressSpace.getNamespace(), 2, queue, new TimeoutBudget(2, TimeUnit.MINUTES));
+        TestUtils.waitForNBrokerReplicas(addressApiClient, kubernetes, messagePersistAddressSpace, 2, queue, new TimeoutBudget(2, TimeUnit.MINUTES));
 
         //send 100000 messages to queue
         UserCredentials user = new UserCredentials("test-change-plan-user", "test_change_plan_pswd");
@@ -471,8 +516,9 @@ class PlansTest extends TestBase implements ISeleniumProviderChrome {
 
         //wait until address will be scaled down to 1 pod
         TestUtils.waitForNBrokerReplicas(
+                addressApiClient,
                 kubernetes,
-                messagePersistAddressSpace.getNamespace(), 1, queue, new TimeoutBudget(2, TimeUnit.MINUTES));
+                messagePersistAddressSpace, 1, queue, new TimeoutBudget(2, TimeUnit.MINUTES));
         //test failed in command above ^, functionality of test code below wasn't verified :) !TODO
 
         //receive messages
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/common/plans/web/ChromeWebConsolePlansTest.java b/systemtests/src/test/java/io/enmasse/systemtest/common/plans/web/ChromeWebConsolePlansTest.java
index 91da3e523..6db01bda4 100644
--- a/systemtests/src/test/java/io/enmasse/systemtest/common/plans/web/ChromeWebConsolePlansTest.java
+++ b/systemtests/src/test/java/io/enmasse/systemtest/common/plans/web/ChromeWebConsolePlansTest.java
@@ -6,10 +6,8 @@ package io.enmasse.systemtest.common.plans.web;
 
 import io.enmasse.systemtest.bases.web.WebConsolePlansTest;
 import io.enmasse.systemtest.selenium.ISeleniumProviderChrome;
-import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
-import org.openqa.selenium.WebDriver;
 
 import static io.enmasse.systemtest.TestTag.nonPR;
 
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/common/upgrade/UpgradeTest.java b/systemtests/src/test/java/io/enmasse/systemtest/common/upgrade/UpgradeTest.java
index cf4097140..7b3d08e19 100644
--- a/systemtests/src/test/java/io/enmasse/systemtest/common/upgrade/UpgradeTest.java
+++ b/systemtests/src/test/java/io/enmasse/systemtest/common/upgrade/UpgradeTest.java
@@ -10,12 +10,11 @@ import io.enmasse.systemtest.amqp.AmqpClient;
 import io.enmasse.systemtest.amqp.ReceiverStatus;
 import io.enmasse.systemtest.bases.TestBase;
 import org.apache.qpid.proton.message.Message;
-import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
 
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.List;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/marathon/MarathonTestBase.java b/systemtests/src/test/java/io/enmasse/systemtest/marathon/MarathonTestBase.java
index 771782153..d5446ab1a 100644
--- a/systemtests/src/test/java/io/enmasse/systemtest/marathon/MarathonTestBase.java
+++ b/systemtests/src/test/java/io/enmasse/systemtest/marathon/MarathonTestBase.java
@@ -45,7 +45,7 @@ abstract class MarathonTestBase extends TestBase {
     // Runner tests methods
     //========================================================================================================
 
-    private void runTestInLoop(int durationMinutes, ITestMethod test) throws Exception {
+    protected void runTestInLoop(int durationMinutes, ITestMethod test) throws Exception {
         log.info(String.format("Starting test running for %d minutes at %s",
                 durationMinutes, new Date().toString()));
         int fails = 0;
@@ -98,7 +98,7 @@ abstract class MarathonTestBase extends TestBase {
 
             log.info("Address space created");
 
-            doAddressTest(addressSpace,"test-topic-createdelete-brokered-%d",
+            doAddressTest(addressSpace, "test-topic-createdelete-brokered-%d",
                     "test-queue-createdelete-brokered-%d", cred);
 
             deleteAddressSpace(addressSpace);
@@ -145,7 +145,7 @@ abstract class MarathonTestBase extends TestBase {
             //check addresses are in ready state
             assertCanConnect(addressSpace, user, queueList);
             assertFalse(res.toString().contains(notReadyString));
-            
+
             deleteAddresses(addressSpace, queueList.toArray(new Destination[0]));
         });
     }
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/marathon/PlansMarathonTest.java b/systemtests/src/test/java/io/enmasse/systemtest/marathon/PlansMarathonTest.java
index 262052130..293306871 100644
--- a/systemtests/src/test/java/io/enmasse/systemtest/marathon/PlansMarathonTest.java
+++ b/systemtests/src/test/java/io/enmasse/systemtest/marathon/PlansMarathonTest.java
@@ -16,7 +16,10 @@ import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
 
 class PlansMarathonTest extends MarathonTestBase {
 
@@ -70,7 +73,9 @@ class PlansMarathonTest extends MarathonTestBase {
         }
         setAddresses(manyAddressesSpace, dest.toArray(new Destination[0]));
 
-        waitForBrokerReplicas(manyAddressesSpace, dest.get(0), 4);
+        for (int i = 0; i < destCount; i += 1000) {
+            waitForBrokerReplicas(manyAddressesSpace, dest.get(i), 1);
+        }
 
         AmqpClient queueClient = amqpClientFactory.createQueueClient(manyAddressesSpace);
         queueClient.getConnectOptions().setCredentials(cred);
@@ -79,7 +84,9 @@ class PlansMarathonTest extends MarathonTestBase {
         }
 
         deleteAddresses(manyAddressesSpace, dest.subList(0, toDeleteCount).toArray(new Destination[0]));
-        waitForBrokerReplicas(manyAddressesSpace, dest.get(0), 2);
+        for (int i = toDeleteCount; i < destCount; i += 1000) {
+            waitForBrokerReplicas(manyAddressesSpace, dest.get(i), 1);
+        }
 
         for (int i = toDeleteCount; i < destCount; i += 50) {
             QueueTest.runQueueTest(queueClient, dest.get(i), 42);
@@ -121,7 +128,9 @@ class PlansMarathonTest extends MarathonTestBase {
 
         appendAddresses(manyAddressesSpace, true, 10, dest.toArray(new Destination[0]));
 
-        waitForBrokerReplicas(manyAddressesSpace, dest.get(0), 4);
+        for (int i = 0; i < destCount; i += 1000) {
+            waitForBrokerReplicas(manyAddressesSpace, dest.get(i), 1);
+        }
 
         AmqpClient queueClient = amqpClientFactory.createQueueClient(manyAddressesSpace);
         queueClient.getConnectOptions().setCredentials(cred);
@@ -130,7 +139,9 @@ class PlansMarathonTest extends MarathonTestBase {
         }
 
         deleteAddresses(manyAddressesSpace, dest.subList(0, toDeleteCount).toArray(new Destination[0]));
-        waitForBrokerReplicas(manyAddressesSpace, dest.get(0), 2);
+        for (int i = toDeleteCount; i < destCount; i += 1000) {
+            waitForBrokerReplicas(manyAddressesSpace, dest.get(i), 1);
+        }
 
         for (int i = toDeleteCount; i < destCount; i += 50) {
             QueueTest.runQueueTest(queueClient, dest.get(i), 42);
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/marathon/RestartTest.java b/systemtests/src/test/java/io/enmasse/systemtest/marathon/RestartTest.java
new file mode 100644
index 000000000..60904d2ec
--- /dev/null
+++ b/systemtests/src/test/java/io/enmasse/systemtest/marathon/RestartTest.java
@@ -0,0 +1,123 @@
+/*
+ * Copyright 2018, EnMasse authors.
+ * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html).
+ */
+package io.enmasse.systemtest.marathon;
+
+import io.enmasse.systemtest.*;
+import io.fabric8.kubernetes.api.model.Pod;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+class RestartTest extends MarathonTestBase {
+    private static Logger log = CustomLogger.getLogger();
+    private ScheduledExecutorService deleteService;
+
+    @BeforeEach
+    void setUp() {
+        deleteService = Executors.newSingleThreadScheduledExecutor();
+    }
+
+    @AfterEach
+    void tearDownRestart() {
+        if (deleteService != null) {
+            deleteService.shutdownNow();
+        }
+    }
+
+    @Test
+    void testRandomDeletePods() throws Exception {
+
+        UserCredentials user = new UserCredentials("test-user", "passsswooooord");
+        AddressSpace standard = new AddressSpace("addr-space-restart-standard", AddressSpaceType.STANDARD, AuthService.STANDARD);
+        AddressSpace brokered = new AddressSpace("addr-space-restart-brokered", AddressSpaceType.BROKERED, AuthService.STANDARD);
+        createAddressSpaceList(standard, brokered);
+        createUser(brokered, user);
+        createUser(standard, user);
+
+        List<Destination> brokeredAddresses = getAllBrokeredAddresses();
+        List<Destination> standardAddresses = getAllStandardAddresses();
+
+        setAddresses(brokered, brokeredAddresses.toArray(new Destination[0]));
+        setAddresses(standard, standardAddresses.toArray(new Destination[0]));
+
+        assertCanConnect(brokered, user, brokeredAddresses);
+        assertCanConnect(standard, user, standardAddresses);
+
+        //set up restart scheduler
+        deleteService.scheduleAtFixedRate(() -> {
+            log.info("............................................................");
+            log.info("............................................................");
+            log.info("..........Scheduler will pick pod and delete them...........");
+            List<Pod> pods = kubernetes.listPods();
+            int podNum = new Random(System.currentTimeMillis()).nextInt(pods.size() - 1);
+            kubernetes.deletePod(environment.namespace(), pods.get(podNum).getMetadata().getName());
+            log.info("............................................................");
+            log.info("............................................................");
+            log.info("............................................................");
+        }, 5, 25, TimeUnit.SECONDS);
+
+        runTestInLoop(60, () ->
+                assertSystemWorks(brokered, standard, user, brokeredAddresses, standardAddresses));
+    }
+
+    @Test
+    @Disabled("Due to issue #2127")
+    void testHAqdrouter() throws Exception {
+
+        UserCredentials user = new UserCredentials("test-user", "passsswooooord");
+        AddressSpace standard = new AddressSpace("addr-space-restart-standard", AddressSpaceType.STANDARD, AuthService.STANDARD);
+        createAddressSpaceList(standard);
+        createUser(standard, user);
+
+        List<Destination> standardAddresses = getAllStandardAddresses();
+
+        setAddresses(standard, standardAddresses.toArray(new Destination[0]));
+
+        assertCanConnect(standard, user, standardAddresses);
+
+        //set up restart scheduler
+        deleteService.scheduleAtFixedRate(() -> {
+            log.info("............................................................");
+            log.info("............................................................");
+            log.info("...........Scheduler will delete one of qdrouter............");
+            List<Pod> qdrouters = kubernetes.listPods().stream().filter(pod -> pod.getMetadata().getName().contains("qdrouter")).collect(Collectors.toList());
+            Pod qdrouter = qdrouters.get(new Random(System.currentTimeMillis()).nextInt(qdrouters.size()) % qdrouters.size());
+            kubernetes.deletePod(environment.namespace(), qdrouter.getMetadata().getName());
+            log.info("............................................................");
+            log.info("............................................................");
+            log.info("............................................................");
+        }, 5, 75, TimeUnit.SECONDS);
+
+        runTestInLoop(30, () ->
+                assertCanConnect(standard, user, standardAddresses));
+    }
+
+    private void assertSystemWorks(AddressSpace brokered, AddressSpace standard, UserCredentials existingUser,
+                                   List<Destination> brAddresses, List<Destination> stAddresses) throws Exception {
+        log.info("Check if system works");
+        TestUtils.runUntilPass(60, () -> getAddressSpace(brokered.getName()));
+        TestUtils.runUntilPass(60, () -> getAddressSpace(standard.getName()));
+        TestUtils.runUntilPass(60, () -> createUser(brokered, new UserCredentials("jenda", "cenda")));
+        TestUtils.runUntilPass(60, () -> createUser(standard, new UserCredentials("jura", "fura")));
+        TestUtils.runUntilPass(60, () -> {
+            assertCanConnect(brokered, existingUser, brAddresses);
+            return true;
+        });
+        TestUtils.runUntilPass(60, () -> {
+            assertCanConnect(standard, existingUser, stAddresses);
+            return true;
+        });
+    }
+}
+
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/standard/AnycastTest.java b/systemtests/src/test/java/io/enmasse/systemtest/standard/AnycastTest.java
index ad9b74086..9ed81e709 100644
--- a/systemtests/src/test/java/io/enmasse/systemtest/standard/AnycastTest.java
+++ b/systemtests/src/test/java/io/enmasse/systemtest/standard/AnycastTest.java
@@ -10,7 +10,6 @@ import io.enmasse.systemtest.ability.ITestBaseStandard;
 import io.enmasse.systemtest.amqp.AmqpClient;
 import io.enmasse.systemtest.bases.TestBaseWithShared;
 import org.apache.qpid.proton.message.Message;
-import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 
 import java.io.IOException;
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/standard/QueueTest.java b/systemtests/src/test/java/io/enmasse/systemtest/standard/QueueTest.java
index 479f99e7d..e7b1fc683 100644
--- a/systemtests/src/test/java/io/enmasse/systemtest/standard/QueueTest.java
+++ b/systemtests/src/test/java/io/enmasse/systemtest/standard/QueueTest.java
@@ -182,7 +182,7 @@ public class QueueTest extends TestBaseWithShared implements ITestBaseStandard {
     void testScaledown() throws Exception {
         Destination dest = Destination.queue("scalequeue", DestinationPlan.STANDARD_LARGE_QUEUE.plan());
         setAddresses(dest);
-        scale(dest, 4);
+        // scale(dest, 4);
 
         Thread.sleep(30000);
         AmqpClient client = amqpClientFactory.createQueueClient();
@@ -207,7 +207,7 @@ public class QueueTest extends TestBaseWithShared implements ITestBaseStandard {
         assertThat("Wrong count of messages received",
                 received.get(1, TimeUnit.MINUTES).size(), is(500));
 
-        scale(dest, 1);
+        // scale(dest, 1);
 
         Thread.sleep(30000);
 
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/standard/SmokeTest.java b/systemtests/src/test/java/io/enmasse/systemtest/standard/SmokeTest.java
index ca0a39c4b..64bed88ad 100644
--- a/systemtests/src/test/java/io/enmasse/systemtest/standard/SmokeTest.java
+++ b/systemtests/src/test/java/io/enmasse/systemtest/standard/SmokeTest.java
@@ -12,14 +12,20 @@ import io.enmasse.systemtest.amqp.AmqpClient;
 import io.enmasse.systemtest.bases.TestBaseWithShared;
 import io.enmasse.systemtest.mqtt.MqttUtils;
 import org.apache.qpid.proton.message.Message;
-import org.eclipse.paho.client.mqttv3.*;
+import org.eclipse.paho.client.mqttv3.IMqttClient;
+import org.eclipse.paho.client.mqttv3.MqttMessage;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 
 import java.nio.charset.StandardCharsets;
-import java.util.*;
-import java.util.concurrent.*;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/standard/TopicTest.java b/systemtests/src/test/java/io/enmasse/systemtest/standard/TopicTest.java
index f24133168..94ddbc055 100644
--- a/systemtests/src/test/java/io/enmasse/systemtest/standard/TopicTest.java
+++ b/systemtests/src/test/java/io/enmasse/systemtest/standard/TopicTest.java
@@ -369,7 +369,7 @@ public class TopicTest extends TestBaseWithShared implements ITestBaseStandard {
         Destination dest = Destination.topic("lrtopic", DestinationPlan.STANDARD_LARGE_TOPIC.plan());
         String linkName = "systest-durable";
         setAddresses(dest);
-        scale(dest, 4);
+        // scale(dest, 4);
 
         Thread.sleep(60_000);
 
@@ -420,7 +420,7 @@ public class TopicTest extends TestBaseWithShared implements ITestBaseStandard {
         log.info("Deploying");
         setAddresses(dest);
         log.info("Scaling");
-        scale(dest, 1);
+        // scale(dest, 1);
 
         Thread.sleep(120_000);
 
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/standard/authz/AuthorizationTest.java b/systemtests/src/test/java/io/enmasse/systemtest/standard/authz/AuthorizationTest.java
index 325771984..f93299a7d 100644
--- a/systemtests/src/test/java/io/enmasse/systemtest/standard/authz/AuthorizationTest.java
+++ b/systemtests/src/test/java/io/enmasse/systemtest/standard/authz/AuthorizationTest.java
@@ -6,7 +6,6 @@ package io.enmasse.systemtest.standard.authz;
 
 import io.enmasse.systemtest.ability.ITestBaseStandard;
 import io.enmasse.systemtest.bases.authz.AuthorizationTestBase;
-import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/standard/clients/proton/java/MsgPatternsInternalTest.java b/systemtests/src/test/java/io/enmasse/systemtest/standard/clients/proton/java/MsgPatternsInternalTest.java
index eb5d8b09e..31be5b58a 100644
--- a/systemtests/src/test/java/io/enmasse/systemtest/standard/clients/proton/java/MsgPatternsInternalTest.java
+++ b/systemtests/src/test/java/io/enmasse/systemtest/standard/clients/proton/java/MsgPatternsInternalTest.java
@@ -8,7 +8,6 @@ import io.enmasse.systemtest.ability.ITestBaseStandard;
 import io.enmasse.systemtest.bases.clients.ClusterClientTestBase;
 import io.enmasse.systemtest.messagingclients.proton.java.ProtonJMSClientReceiver;
 import io.enmasse.systemtest.messagingclients.proton.java.ProtonJMSClientSender;
-import io.enmasse.systemtest.messagingclients.proton.python.PythonClientReceiver;
 import org.junit.jupiter.api.Test;
 
 class MsgPatternsInternalTest extends ClusterClientTestBase implements ITestBaseStandard {
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/standard/mqtt/InteroperabilityTest.java b/systemtests/src/test/java/io/enmasse/systemtest/standard/mqtt/InteroperabilityTest.java
index 7d4f7c601..0c80c4059 100644
--- a/systemtests/src/test/java/io/enmasse/systemtest/standard/mqtt/InteroperabilityTest.java
+++ b/systemtests/src/test/java/io/enmasse/systemtest/standard/mqtt/InteroperabilityTest.java
@@ -16,7 +16,8 @@ import org.apache.qpid.proton.amqp.Binary;
 import org.apache.qpid.proton.amqp.messaging.AmqpValue;
 import org.apache.qpid.proton.amqp.messaging.Data;
 import org.apache.qpid.proton.message.Message;
-import org.eclipse.paho.client.mqttv3.*;
+import org.eclipse.paho.client.mqttv3.IMqttClient;
+import org.eclipse.paho.client.mqttv3.MqttMessage;
 import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
 
@@ -58,7 +59,7 @@ class InteroperabilityTest extends TestBaseWithShared implements ITestBaseStanda
         AmqpClient amqpClient = amqpClientFactory.createTopicClient();
         Future<List<Message>> recvResultAmqp = amqpClient.recvMessages(mqttTopic.getAddress(), messages.size());
 
-        List<CompletableFuture<Void>> publishFutures = MqttUtils.publish(mqttClient, mqttTopic.getAddress(), messages);;
+        List<CompletableFuture<Void>> publishFutures = MqttUtils.publish(mqttClient, mqttTopic.getAddress(), messages);
 
         int sentCount = MqttUtils.awaitAndReturnCode(publishFutures, 1, TimeUnit.MINUTES);
         assertThat("Incorrect count of messages sent",
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/standard/mqtt/SessionTest.java b/systemtests/src/test/java/io/enmasse/systemtest/standard/mqtt/SessionTest.java
index 16db21dba..af7920fdd 100644
--- a/systemtests/src/test/java/io/enmasse/systemtest/standard/mqtt/SessionTest.java
+++ b/systemtests/src/test/java/io/enmasse/systemtest/standard/mqtt/SessionTest.java
@@ -12,7 +12,8 @@ import io.enmasse.systemtest.ability.ITestBaseStandard;
 import io.enmasse.systemtest.bases.TestBaseWithShared;
 import io.enmasse.systemtest.mqtt.MqttDeliveryCompleteCallback;
 import io.enmasse.systemtest.mqtt.MqttMessageArrivedCallback;
-import org.eclipse.paho.client.mqttv3.*;
+import org.eclipse.paho.client.mqttv3.IMqttClient;
+import org.eclipse.paho.client.mqttv3.MqttConnectOptions;
 import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
 
@@ -50,8 +51,8 @@ public class SessionTest extends TestBaseWithShared implements ITestBaseStandard
 
         // Subscribe on a cleanSession false and disconnect
         IMqttClient subscriber = mqttClientFactory.build().clientId(SUBSCRIBER_ID)
-                                                           .mqttConnectionOptions(persistentSession)
-                                                           .create();
+                .mqttConnectionOptions(persistentSession)
+                .create();
         subscriber.connect();
         subscriber.subscribe(MQTT_TOPIC, 1);
         subscriber.disconnect();
@@ -62,8 +63,8 @@ public class SessionTest extends TestBaseWithShared implements ITestBaseStandard
         // Connect the subscriber again, with cleanSession false, expect the message to arrive
         CompletableFuture<Void> messageArrived = new CompletableFuture<>();
         subscriber = mqttClientFactory.build().clientId(SUBSCRIBER_ID)
-                                               .mqttConnectionOptions(persistentSession)
-                                               .create();
+                .mqttConnectionOptions(persistentSession)
+                .create();
         subscriber.setCallback((MqttMessageArrivedCallback) (topic, message) -> messageArrived.complete(null));
         subscriber.connect();
         messageArrived.get();
@@ -77,8 +78,8 @@ public class SessionTest extends TestBaseWithShared implements ITestBaseStandard
 
         AtomicInteger receivedMessageCount = new AtomicInteger();
         subscriber = mqttClientFactory.build().clientId(SUBSCRIBER_ID)
-                                               .mqttConnectionOptions(cleanSession)
-                                               .create();
+                .mqttConnectionOptions(cleanSession)
+                .create();
         subscriber.setCallback((MqttMessageArrivedCallback) (topic, message) -> receivedMessageCount.incrementAndGet());
         subscriber.connect();
         // TODO give reasonable time for the message not to arrive
@@ -89,8 +90,8 @@ public class SessionTest extends TestBaseWithShared implements ITestBaseStandard
         // Connect the subscriber again, with cleanSession true, publish and check the message does not arrive so
         // proving that the subscription was not persisted.
         subscriber = mqttClientFactory.build().clientId(SUBSCRIBER_ID)
-                                               .mqttConnectionOptions(cleanSession)
-                                               .create();
+                .mqttConnectionOptions(cleanSession)
+                .create();
         subscriber.setCallback((MqttMessageArrivedCallback) (topic, message) -> receivedMessageCount.incrementAndGet());
         subscriber.connect();
         Thread.sleep(2500);
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/standard/web/ChromeWebSocketBrowserTest.java b/systemtests/src/test/java/io/enmasse/systemtest/standard/web/ChromeWebSocketBrowserTest.java
index 3b4b618e6..a38e10d9b 100644
--- a/systemtests/src/test/java/io/enmasse/systemtest/standard/web/ChromeWebSocketBrowserTest.java
+++ b/systemtests/src/test/java/io/enmasse/systemtest/standard/web/ChromeWebSocketBrowserTest.java
@@ -4,11 +4,11 @@
  */
 package io.enmasse.systemtest.standard.web;
 
-import io.enmasse.systemtest.*;
+import io.enmasse.systemtest.AddressType;
+import io.enmasse.systemtest.Destination;
 import io.enmasse.systemtest.ability.ITestBaseStandard;
 import io.enmasse.systemtest.bases.web.WebSocketBrowserTest;
 import io.enmasse.systemtest.selenium.ISeleniumProviderChrome;
-import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/standard/web/FirefoxWebConsoleTest.java b/systemtests/src/test/java/io/enmasse/systemtest/standard/web/FirefoxWebConsoleTest.java
index be113e505..c19b9cbba 100644
--- a/systemtests/src/test/java/io/enmasse/systemtest/standard/web/FirefoxWebConsoleTest.java
+++ b/systemtests/src/test/java/io/enmasse/systemtest/standard/web/FirefoxWebConsoleTest.java
@@ -193,7 +193,8 @@ public class FirefoxWebConsoleTest extends WebConsoleTest implements ITestBaseSt
     }
 
     @Test
-    @Disabled("disabled while sdavey changes it with changes to regex in addr names") //TODO(sdavey)
+    @Disabled("disabled while sdavey changes it with changes to regex in addr names")
+        //TODO(sdavey)
     void testCreateAddressWithSymbolsAt61stCharIndex() throws Exception {
         doTestCreateAddressWithSymbolsAt61stCharIndex(
                 Destination.queue("queue10charHere-10charHere-10charHere-10charHere-10charHere-1",
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/standard/web/FirefoxWebSocketBrowserTest.java b/systemtests/src/test/java/io/enmasse/systemtest/standard/web/FirefoxWebSocketBrowserTest.java
index dca578441..387fd68d9 100644
--- a/systemtests/src/test/java/io/enmasse/systemtest/standard/web/FirefoxWebSocketBrowserTest.java
+++ b/systemtests/src/test/java/io/enmasse/systemtest/standard/web/FirefoxWebSocketBrowserTest.java
@@ -9,7 +9,6 @@ import io.enmasse.systemtest.Destination;
 import io.enmasse.systemtest.ability.ITestBaseStandard;
 import io.enmasse.systemtest.bases.web.WebSocketBrowserTest;
 import io.enmasse.systemtest.selenium.ISeleniumProviderFirefox;
-import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 
 class FirefoxWebSocketBrowserTest extends WebSocketBrowserTest implements ITestBaseStandard, ISeleniumProviderFirefox {