Repository: EnMasseProject/enmasse
Pull request #3276 (2019-10-07 11:03:50)

improve testMessagingDuringRestartComponents
### Description

this PR reimplements the test CommonTest#testMessagingDuringRestartComponents in order to consume less resources because we have seen cases in which this test fails because of the recursion used

### Checklist

_Please go through this checklist and make sure all applicable tasks have been done_

- [ ] Update/write design documentation in `./documentation/design`
- [ ] Write tests and make sure they pass
- [ ] Update documentation
- [ ] Check RBAC rights for Kubernetes / OpenShift roles
- [ ] Try your changes from Pod inside your Kubernetes and OpenShift cluster, not just locally
- [ ] Reference relevant issue(s) and close them after merging
- [ ] Update CHANGELOG.md


famartinrh: @enmasse-ci testcase=CommonTest#testMessagingDuringRestartComponents profile=isolated
famartinrh: @enmasse-ci testcase=CommonTest#testMessagingDuringRestartComponents profile=isolated
famartinrh: @enmasse-ci run tests testcase=CommonTest#testMessagingDuringRestartComponents profile=isolated
enmasse-ci: ### :heavy_check_mark: Test Summary :heavy_check_mark:
**TEST_PROFILE**: isolated
**TEST_CASE:** CommonTest#testMessagingDuringRestartComponents
**TOTAL:** 1
**PASS:** 1
**FAIL:** 0
**SKIP:** 0


Patch:
diff --git a/.travis.yml b/.travis.yml
index 1c5ddc041..f1c7b70ce 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -30,7 +30,6 @@ jobs:
     before_install:
     - ./.travis/find-qdrouterd-base-versions.sh
     - pip install --user requests[security] ansible
-    - echo "MAVEN_OPTS='-Xmx2g -XX:MaxPermSize=512m'" > ~/.mavenrc
     - gem install asciidoctor
     - gimme 1.12.9
     - source ~/.gimme/envs/go1.12.9.env
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1b782b5a4..11640e584 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,7 +1,15 @@
 ## 0.30.0
 * Fix go tool vet on newer versions of go
 * #3247: Fix path to probe endpoint used when running on Kubernetes
-
+* Add support for LoRaWAN and Sigfox
+* Deploy IoT operator automatically when the IoTConfig is created
+* Drop IoT garbage collector deployment in favor of using finalizers to clean up
+* Implement device registry authentication based on Kubernetes user
+* Upgrade to Apache ActiveMQ Artemis 2.10.0
+* Upgrade to Apache Qpid Dispatch Router 1.9.0
+* Add support for connecting address spaces to external AMQP endpoints (standard address space)
+* Add support for forwarding messages between external endpoints and local queues (standard address space)
+* Add scripts for performing controlled rolling restart
 
 ## 0.29.0
 * Upgrade to Apache ActiveMQ Artemis 2.9.0
diff --git a/HACKING.md b/HACKING.md
index 8f0a22102..bfb198dd2 100644
--- a/HACKING.md
+++ b/HACKING.md
@@ -98,17 +98,13 @@ oc apply -f templates/build/enmasse-latest/install/components/example-authservic
 
 ### Running full systemtest suite
 
-#### Install systemtests dependencies
-
-    ansible-playbook systemtests/ansible/playbooks/systemtests-dependencies.yml --tags clients --skip-tags dependencies
-
 #### Running the systemtests
 
     make systemtests
 
 #### Run single system test
 
-    make SYSTEMTEST_ARGS="io.enmasse.systemtest.standard.QueueTest#testCreateDeleteQueue" systemtests
+    make SYSTEMTEST_ARGS="shared.standard.QueueTest#testCreateDeleteQueue" systemtests
 
 ### Adding / Updating go dependencies
 
diff --git a/Makefile.env.mk b/Makefile.env.mk
index 462d8b9df..9cac12f90 100644
--- a/Makefile.env.mk
+++ b/Makefile.env.mk
@@ -20,7 +20,7 @@ IMAGE_VERSION          ?= $(TAG)
 ADDRESS_SPACE_CONTROLLER_IMAGE ?= $(DOCKER_REGISTRY_PREFIX)$(DOCKER_ORG)/address-space-controller:$(IMAGE_VERSION)
 API_SERVER_IMAGE ?= $(DOCKER_REGISTRY_PREFIX)$(DOCKER_ORG)/api-server:$(IMAGE_VERSION)
 STANDARD_CONTROLLER_IMAGE ?= $(DOCKER_REGISTRY_PREFIX)$(DOCKER_ORG)/standard-controller:$(IMAGE_VERSION)
-ROUTER_IMAGE ?= quay.io/interconnectedcloud/qdrouterd:1.8.0
+ROUTER_IMAGE ?= quay.io/interconnectedcloud/qdrouterd:1.9.0
 BROKER_PLUGIN_IMAGE ?= $(DOCKER_REGISTRY_PREFIX)$(DOCKER_ORG)/broker-plugin:$(IMAGE_VERSION)
 TOPIC_FORWARDER_IMAGE ?= $(DOCKER_REGISTRY_PREFIX)$(DOCKER_ORG)/topic-forwarder:$(IMAGE_VERSION)
 AGENT_IMAGE ?= $(DOCKER_REGISTRY_PREFIX)$(DOCKER_ORG)/agent:$(IMAGE_VERSION)
diff --git a/address-space-controller/src/main/java/io/enmasse/controller/CreateController.java b/address-space-controller/src/main/java/io/enmasse/controller/CreateController.java
index fe535c248..b7deb26c8 100644
--- a/address-space-controller/src/main/java/io/enmasse/controller/CreateController.java
+++ b/address-space-controller/src/main/java/io/enmasse/controller/CreateController.java
@@ -5,7 +5,6 @@
 package io.enmasse.controller;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
 import io.enmasse.address.model.*;
 import io.enmasse.admin.model.AddressPlan;
 import io.enmasse.admin.model.AddressSpacePlan;
@@ -36,7 +35,6 @@ import static io.enmasse.k8s.api.EventLogger.Type.Normal;
 public class CreateController implements Controller {
     private static final Logger log = LoggerFactory.getLogger(CreateController.class.getName());
 
-    private static final ObjectMapper mapper = new ObjectMapper();
     private final Kubernetes kubernetes;
     private final SchemaProvider schemaProvider;
     private final InfraResourceFactory infraResourceFactory;
diff --git a/api-model/src/main/java/io/enmasse/iot/model/v1/CommonCondition.java b/api-model/src/main/java/io/enmasse/iot/model/v1/CommonCondition.java
new file mode 100644
index 000000000..8b1021bd8
--- /dev/null
+++ b/api-model/src/main/java/io/enmasse/iot/model/v1/CommonCondition.java
@@ -0,0 +1,86 @@
+/*
+ * Copyright 2019, EnMasse authors.
+ * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html).
+ */
+
+package io.enmasse.iot.model.v1;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.google.common.base.MoreObjects;
+import com.google.common.base.MoreObjects.ToStringHelper;
+
+import io.fabric8.kubernetes.api.model.Doneable;
+import io.sundr.builder.annotations.Buildable;
+import io.sundr.builder.annotations.Inline;
+
+@Buildable(
+        editableEnabled = false,
+        generateBuilderPackage = false,
+        builderPackage = "io.fabric8.kubernetes.api.builder",
+        inline = @Inline(
+                type = Doneable.class,
+                prefix = "Doneable",
+                value = "done"))
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class CommonCondition<T extends Enum<T>> {
+
+    private T type;
+    private String status;
+    private String lastTransitionTime;
+    private String reason;
+    private String message;
+
+    public void setType(T type) {
+        this.type = type;
+    }
+
+    public T getType() {
+        return type;
+    }
+
+    public String getStatus() {
+        return status;
+    }
+
+    public void setStatus(String status) {
+        this.status = status;
+    }
+
+    public String getLastTransitionTime() {
+        return lastTransitionTime;
+    }
+
+    public void setLastTransitionTime(String lastTransitionTime) {
+        this.lastTransitionTime = lastTransitionTime;
+    }
+
+    public String getReason() {
+        return reason;
+    }
+
+    public void setReason(String reason) {
+        this.reason = reason;
+    }
+
+    public String getMessage() {
+        return message;
+    }
+
+    public void setMessage(String message) {
+        this.message = message;
+    }
+
+    protected ToStringHelper toStringHelper() {
+        return MoreObjects.toStringHelper(this)
+                .add("type", this.type)
+                .add("status", this.status)
+                .add("lastTransitionTime", this.lastTransitionTime)
+                .add("reason", this.reason)
+                .add("message", this.message);
+    }
+
+    @Override
+    public String toString() {
+        return toStringHelper().toString();
+    }
+}
diff --git a/api-model/src/main/java/io/enmasse/iot/model/v1/ConfigCondition.java b/api-model/src/main/java/io/enmasse/iot/model/v1/ConfigCondition.java
new file mode 100644
index 000000000..7f8cd6f8a
--- /dev/null
+++ b/api-model/src/main/java/io/enmasse/iot/model/v1/ConfigCondition.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2019, EnMasse authors.
+ * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html).
+ */
+
+package io.enmasse.iot.model.v1;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+
+import io.fabric8.kubernetes.api.model.Doneable;
+import io.sundr.builder.annotations.Buildable;
+import io.sundr.builder.annotations.Inline;
+
+@Buildable(
+        editableEnabled = false,
+        generateBuilderPackage = false,
+        builderPackage = "io.fabric8.kubernetes.api.builder",
+        inline = @Inline(
+                type = Doneable.class,
+                prefix = "Doneable",
+                value = "done"))
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class ConfigCondition extends CommonCondition<ProjectConditionType> {
+
+}
diff --git a/api-model/src/main/java/io/enmasse/iot/model/v1/ConfigConditionType.java b/api-model/src/main/java/io/enmasse/iot/model/v1/ConfigConditionType.java
new file mode 100644
index 000000000..5fec1dd24
--- /dev/null
+++ b/api-model/src/main/java/io/enmasse/iot/model/v1/ConfigConditionType.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2019, EnMasse authors.
+ * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html).
+ */
+
+package io.enmasse.iot.model.v1;
+
+import com.fasterxml.jackson.annotation.JsonValue;
+
+public enum ConfigConditionType {
+    READY("Ready");
+
+    @JsonValue
+    final private String value;
+
+    private ConfigConditionType(final String value) {
+        this.value = value;
+    }
+
+    public String getValue() {
+        return value;
+    }
+}
diff --git a/api-model/src/main/java/io/enmasse/iot/model/v1/IoTConfigStatus.java b/api-model/src/main/java/io/enmasse/iot/model/v1/IoTConfigStatus.java
index dee492648..a4e3b0ad9 100644
--- a/api-model/src/main/java/io/enmasse/iot/model/v1/IoTConfigStatus.java
+++ b/api-model/src/main/java/io/enmasse/iot/model/v1/IoTConfigStatus.java
@@ -4,6 +4,7 @@
  */
 package io.enmasse.iot.model.v1;
 
+import java.util.List;
 import java.util.Map;
 
 import com.fasterxml.jackson.annotation.JsonInclude;
@@ -26,10 +27,12 @@ import io.sundr.builder.annotations.Inline;
 public class IoTConfigStatus {
 
     private boolean initialized = false;
-    private String state;
+    private String phase;
+    private String phaseReason;
     private String authenticationServicePSK;
     private Map<String, AdapterStatus> adapters;
     private Map<String, ServiceStatus> services;
+    private List<ConfigCondition> conditions;
 
     public boolean isInitialized() {
         return initialized;
@@ -39,12 +42,20 @@ public class IoTConfigStatus {
         this.initialized = initialized;
     }
 
-    public String getState() {
-        return state;
+    public String getPhase() {
+        return phase;
     }
 
-    public void setState(String state) {
-        this.state = state;
+    public void setPhase(String phase) {
+        this.phase = phase;
+    }
+
+    public String getPhaseReason() {
+        return phaseReason;
+    }
+
+    public void setPhaseReason(String phaseReason) {
+        this.phaseReason = phaseReason;
     }
 
     public String getAuthenticationServicePSK() {
@@ -71,4 +82,12 @@ public class IoTConfigStatus {
         this.services = services;
     }
 
+    public List<ConfigCondition> getConditions() {
+        return conditions;
+    }
+
+    public void setConditions(List<ConfigCondition> conditions) {
+        this.conditions = conditions;
+    }
+
 }
diff --git a/api-model/src/main/java/io/enmasse/iot/model/v1/IoTProjectStatus.java b/api-model/src/main/java/io/enmasse/iot/model/v1/IoTProjectStatus.java
index d24f912c2..e3a434150 100644
--- a/api-model/src/main/java/io/enmasse/iot/model/v1/IoTProjectStatus.java
+++ b/api-model/src/main/java/io/enmasse/iot/model/v1/IoTProjectStatus.java
@@ -4,6 +4,8 @@
  */
 package io.enmasse.iot.model.v1;
 
+import java.util.List;
+
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
@@ -31,6 +33,7 @@ public class IoTProjectStatus {
     private boolean ready = false;
     private String tenantName;
     private ExternalDownstreamStrategy downstreamEndpoint;
+    private List<ProjectCondition> conditions;
 
     public boolean isReady() {
         return ready;
@@ -56,4 +59,12 @@ public class IoTProjectStatus {
         this.downstreamEndpoint = downstreamEndpoint;
     }
 
+    public List<ProjectCondition> getConditions() {
+        return conditions;
+    }
+
+    public void setConditions(List<ProjectCondition> conditions) {
+        this.conditions = conditions;
+    }
+
 }
diff --git a/api-model/src/main/java/io/enmasse/iot/model/v1/ProjectCondition.java b/api-model/src/main/java/io/enmasse/iot/model/v1/ProjectCondition.java
new file mode 100644
index 000000000..2385d49b8
--- /dev/null
+++ b/api-model/src/main/java/io/enmasse/iot/model/v1/ProjectCondition.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2019, EnMasse authors.
+ * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html).
+ */
+
+package io.enmasse.iot.model.v1;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+
+import io.fabric8.kubernetes.api.model.Doneable;
+import io.sundr.builder.annotations.Buildable;
+import io.sundr.builder.annotations.Inline;
+
+@Buildable(
+        editableEnabled = false,
+        generateBuilderPackage = false,
+        builderPackage = "io.fabric8.kubernetes.api.builder",
+        inline = @Inline(
+                type = Doneable.class,
+                prefix = "Doneable",
+                value = "done"))
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class ProjectCondition extends CommonCondition<ProjectConditionType> {
+
+}
diff --git a/api-model/src/main/java/io/enmasse/iot/model/v1/ProjectConditionType.java b/api-model/src/main/java/io/enmasse/iot/model/v1/ProjectConditionType.java
new file mode 100644
index 000000000..0fbe22919
--- /dev/null
+++ b/api-model/src/main/java/io/enmasse/iot/model/v1/ProjectConditionType.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright 2019, EnMasse authors.
+ * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html).
+ */
+
+package io.enmasse.iot.model.v1;
+
+import com.fasterxml.jackson.annotation.JsonValue;
+
+public enum ProjectConditionType {
+    READY("Ready"),
+    RESOURCES_CREATED("ResourcesCreated"),
+    RESOURCES_READY("ResourcesReady"),
+    ;
+
+    @JsonValue
+    private final String value;
+
+    private ProjectConditionType(final String value) {
+        this.value = value;
+    }
+
+    public String getValue() {
+        return value;
+    }
+}
diff --git a/documentation/assemblies/assembly-configuring.adoc b/documentation/assemblies/assembly-configuring.adoc
index 76469091c..fb3208f58 100644
--- a/documentation/assemblies/assembly-configuring.adoc
+++ b/documentation/assemblies/assembly-configuring.adoc
@@ -25,3 +25,10 @@ include::../assemblies/assembly-auth-services.adoc[leveloffset=+1]
 
 include::../modules/ref-example-roles.adoc[leveloffset=+1]
 
+ifeval::["{cmdcli}" == "kubectl"]
+include::../modules/proc-configure-openid-connect-for-kubernetes.adoc[leveloffset=+1]
+endif::[]
+
+
+
+
diff --git a/documentation/assemblies/assembly-using-console.adoc b/documentation/assemblies/assembly-using-console.adoc
index f1da96916..590759aba 100644
--- a/documentation/assemblies/assembly-using-console.adoc
+++ b/documentation/assemblies/assembly-using-console.adoc
@@ -7,6 +7,11 @@
 [id='assembly-using-console-{context}']
 = Using the {ConsoleName}
 
+ifeval::["{cmdcli}" == "kubectl"]
+.Prerequisites
+* You must have configured Kubernetes and the {Product Name} Console to use OpenID Connect. For more information see link:{BookUrlBase}{BaseProductVersion}{BookNameUrl}#config-openid-connect-for-kubernetes[Configuring the EnMasse Console to use OpenID Connect].
+endif::[]
+
 You can use the {ConsoleName} to perform tasks such as link:{BookUrlBase}{BaseProductVersion}{BookNameUrl}#proc-create-address-space-console-messaging[creating] and link:{BookUrlBase}{BaseProductVersion}{BookNameUrl}#proc-delete-address-space-console-messaging[deleting an address space], link:{BookUrlBase}{BaseProductVersion}{BookNameUrl}#create-address-console-messaging[creating an address], and link:{BookUrlBase}{BaseProductVersion}{BookNameUrl}#ref-view-message-connection-stats-table-messaging[viewing message and connection statistics].
 
 include::../modules/con-console.adoc[leveloffset=+1]
diff --git a/documentation/modules/con-standard-queue.adoc b/documentation/modules/con-standard-queue.adoc
index 2ce9f8f48..5bf4ad484 100644
--- a/documentation/modules/con-standard-queue.adoc
+++ b/documentation/modules/con-standard-queue.adoc
@@ -6,6 +6,8 @@
 = Queue
 // !standard.address.queue.shortDescription:A store-and-forward queue
 // !standard.address.queue.longDescription:start
-The queue address type is a store-and-forward queue. This address type is appropriate for implementing a distributed work queue, handling traffic bursts, and other use cases where you want to decouple the producer and consumer. A queue can be sharded across multiple storage units. Message ordering might be lost for queues in the standard address space.
+The queue address type is a store-and-forward queue. This address type is appropriate for implementing a distributed work queue, handling traffic bursts, and other use cases when you want to decouple the producer and consumer. A queue can be sharded across multiple storage units. Message ordering might be lost for queues in the standard address space.
 // !standard.address.queue.longDescription:stop
 
+Regarding dead-letter queue (DLQ) storing, you can determine if any stored DLQs exist by logging in to the console and link:{BookUrlBase}{BaseProductVersion}{BookNameUrl}#ref-view-message-conn-stats-table-messaging[viewing the Addresses page]. To resolve this situation, you must connect with a client and consume from a DLQ address.
+
diff --git a/documentation/modules/proc-configure-openid-connect-for-kubernetes.adoc b/documentation/modules/proc-configure-openid-connect-for-kubernetes.adoc
new file mode 100644
index 000000000..b172256d9
--- /dev/null
+++ b/documentation/modules/proc-configure-openid-connect-for-kubernetes.adoc
@@ -0,0 +1,64 @@
+// Module included in the following assemblies:
+//
+// assembly-configuring.adoc
+
+[id='config-openid-connect-for-kubernetes']
+= Configuring {ProductName} Console to use OpenID Connect
+
+To use the {ProductName} Console on Kubernetes, you must configure Kubernetes to use
+https://kubernetes.io/docs/reference/access-authn-authz/authentication/#openid-connect-tokens[OpenID Connect (OIDC)]
+as an Authentication Strategy.  Then you must create a `consoleservice` resource refering to your OIDC Provider.
+
+
+.Prerequestises
+* Before you begin you need to know the following details from your OpenID Connect provider:
+** OIDC Discovery URL
+** OIDC scopes
+** Client ID
+** Client secret
++
+NOTE: If using a public OIDC provider (such as Google, Azure, GitHub, etc) the
+https://pusher.github.io/oauth2_proxy/auth-configuration[OAuthProxy configuration guide] offers specific guidance.
+
+* The Kubernetes api-server must be configured to use the
+https://kubernetes.io/docs/reference/access-authn-authz/authentication/#configuring-the-api-server[OpenID Connect plugin].
++
+If you are using minikube, see  https://minikube.sigs.k8s.io/docs/tutorials/openid_connect_auth/[these instructions].
+will guide you.
+
+.Procedure
+
+. Select the namespace where {ProductName} is installed:
++
+[subs="+quotes,attributes",options="nowrap"]
+----
+{cmdcli} config set-context $(kubectl config current-context) --namespace=_{ProductNamespace}_
+----
+
+. Create a secret definition with the client-id/client-secret pair of your OIDC provider:
++
+[options="nowrap",subs="attributes"]
+----
+{cmdcli} create secret generic my-google-oidc-secret --from-literal=client-id=myclientid --from-literal=client-secret=mysecret
+----
+
+. Create a console services definition:
++
+[options="nowrap",subs="+quotes,attributes"]
+----
+cat <<EOF | {cmdcli} apply -f -
+apiVersion: admin.enmasse.io/v1beta1
+kind: ConsoleService
+metadata:
+    name: console
+spec:
+    discoveryMetadataURL: https://accounts.google.com/.well-known/openid-configuration
+    oauthClientSecret:
+        name: my-google-oidc-secret
+    scope: openid email
+EOF
+----
++
+NOTE: Replace the discovery URL and scopes with the appropriate values from your OIDC provider.  Ensure that
+oauthClientSecret references the secret created in the previous step.
+
diff --git a/documentation/modules/ref-view-message-conn-stats-table.adoc b/documentation/modules/ref-view-message-conn-stats-table.adoc
index 8db090aee..238e1af79 100644
--- a/documentation/modules/ref-view-message-conn-stats-table.adoc
+++ b/documentation/modules/ref-view-message-conn-stats-table.adoc
@@ -21,6 +21,7 @@
 |Number of receivers attached |*Receivers*
 |Queue and topic address types only: Number of stored messages on the broker or brokers |*Stored*
 |Standard address space only: Message deliveries per second |For the desired address, expand the twisty on the left to show the *Senders* table; see the *Delivery Rate* column.
+|Standard address space and queue address type only: Number of dead letter queues (DLQs) |*Stored DLQ*
 |===
 
 
diff --git a/iot/iot-device-registry-infinispan/src/main/java/io/enmasse/iot/registry/infinispan/device/impl/DeviceManagementServiceImpl.java b/iot/iot-device-registry-infinispan/src/main/java/io/enmasse/iot/registry/infinispan/device/impl/DeviceManagementServiceImpl.java
index 2f6709b08..23935ded7 100644
--- a/iot/iot-device-registry-infinispan/src/main/java/io/enmasse/iot/registry/infinispan/device/impl/DeviceManagementServiceImpl.java
+++ b/iot/iot-device-registry-infinispan/src/main/java/io/enmasse/iot/registry/infinispan/device/impl/DeviceManagementServiceImpl.java
@@ -22,6 +22,7 @@ import org.eclipse.hono.service.management.Id;
 import org.eclipse.hono.service.management.OperationResult;
 import org.eclipse.hono.service.management.Result;
 import org.eclipse.hono.service.management.device.Device;
+import org.infinispan.client.hotrod.Flag;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -54,6 +55,7 @@ public class DeviceManagementServiceImpl extends AbstractDeviceManagementService
         value.setRegistrationInformation(Json.encode(device));
 
         return this.managementCache
+                .withFlags(Flag.FORCE_RETURN_VALUE)
                 .putIfAbsentAsync(key, value)
                 .thenApply(result -> {
                     if (result == null) {
diff --git a/keycloak-user-api/src/main/java/io/enmasse/user/keycloak/KeycloakUserApi.java b/keycloak-user-api/src/main/java/io/enmasse/user/keycloak/KeycloakUserApi.java
index 3e58c4039..4c16c6167 100644
--- a/keycloak-user-api/src/main/java/io/enmasse/user/keycloak/KeycloakUserApi.java
+++ b/keycloak-user-api/src/main/java/io/enmasse/user/keycloak/KeycloakUserApi.java
@@ -34,18 +34,42 @@ import java.time.Duration;
 import java.time.Instant;
 import java.util.*;
 import java.util.function.Predicate;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 import static java.util.Optional.empty;
 
-
 public class KeycloakUserApi implements UserApi, RealmApi {
 
     private static final String ATTR_ANNOTATIONS = "annotations";
     private static final String ATTR_OWNER_REFERENCES = "ownerReferences";
 
+    private static final String BASE = "io.enmasse.user.keycloak";
+    private static final int MAX_ANNOTATION_VALUE_LENGTH = Integer.getInteger(BASE + ".maxUserAnnotationValueLength", 255);
+
     private static final Logger log = LoggerFactory.getLogger(KeycloakUserApi.class);
 
+    private static final String DEFAULT_ANNOTATION_BLACKLIST = "";
+    private static final String DEFAULT_ANNOTATION_WHITELIST = "(.*\\.|)enmasse\\.io/.*";
+
+    private static final List<Pattern> ANNOTATION_BLACKLIST;
+    private static final List<Pattern> ANNOTATION_WHITELIST;
+
+    static {
+        ANNOTATION_BLACKLIST = Arrays.stream(System.getProperty(BASE + ".annotationBlacklist", DEFAULT_ANNOTATION_BLACKLIST)
+                .split(","))
+                .filter(Predicate.not(String::isBlank))
+                .map(Pattern::compile)
+                .collect(Collectors.toUnmodifiableList());
+        ANNOTATION_WHITELIST = Arrays.stream(System.getProperty(BASE + ".annotationWhitelist", DEFAULT_ANNOTATION_WHITELIST)
+                .split(","))
+                .filter(Predicate.not(String::isBlank))
+                .map(Pattern::compile)
+                .collect(Collectors.toUnmodifiableList());
+        log.info("Annotation blacklist: {}", ANNOTATION_BLACKLIST);
+        log.info("Annotation whitelist: {}", ANNOTATION_WHITELIST);
+    }
+
     private final Clock clock;
     private final KeycloakFactory keycloakFactory;
     private final Duration apiTimeout;
@@ -184,6 +208,7 @@ public class KeycloakUserApi implements UserApi, RealmApi {
         final ObjectMapper mapper = new ObjectMapper();
 
         return annotations.entrySet().stream()
+                .filter(KeycloakUserApi::filterUserAnnotation)
                 .map(entry -> {
                     try {
                         return mapper.writeValueAsString(entry);
@@ -238,7 +263,7 @@ public class KeycloakUserApi implements UserApi, RealmApi {
 
     private static List<OwnerReference> ownerReferencesFromString(final List<String> attributeValues) {
 
-        if (attributeValues == null || attributeValues.isEmpty() ) {
+        if (attributeValues == null || attributeValues.isEmpty()) {
             return null;
         }
 
@@ -267,8 +292,13 @@ public class KeycloakUserApi implements UserApi, RealmApi {
 
     @Override
     public void createUser(AuthenticationService authenticationService, String realmName, User user) throws Exception {
-        log.info("Creating user {} in realm {}", user.getSpec().getUsername(), realmName);
+        Objects.requireNonNull(authenticationService);
+        Objects.requireNonNull(realmName);
+        Objects.requireNonNull(user);
+
         user.validate();
+
+        log.info("Creating user {} in realm {}", user.getSpec().getUsername(), realmName);
         validateForCreation(user);
 
         withRealm(authenticationService, realmName, realm -> {
@@ -316,6 +346,7 @@ public class KeycloakUserApi implements UserApi, RealmApi {
 
     /**
      * Check if the user is valid for creating a new instance.
+     *
      * @param user The user to check.
      */
     private void validateForCreation(final User user) {
@@ -456,11 +487,11 @@ public class KeycloakUserApi implements UserApi, RealmApi {
 
         final String[] currentValue = Optional.ofNullable(userRep.getAttributes().get(key))
                 .map(v -> v.toArray(String[]::new))
-                .orElseGet(()-> new String[] {});
+                .orElseGet(() -> new String[] {});
 
         final String[] newValue = Optional.ofNullable(value)
-                .map(v-> v.toArray(String[]::new))
-                .orElseGet(()-> new String[] {});
+                .map(v -> v.toArray(String[]::new))
+                .orElseGet(() -> new String[] {});
 
         if (Arrays.equals(currentValue, newValue)) {
             return false;
@@ -555,7 +586,8 @@ public class KeycloakUserApi implements UserApi, RealmApi {
         return withKeycloak(authenticationService, kc -> getRealmResource(kc, realmName) != null);
     }
 
-    private UserList queryUsers(AuthenticationService authenticationService, final Predicate<RealmRepresentation> realmPredicate, final Predicate<UserRepresentation> userPredicate) throws Exception {
+    private UserList queryUsers(AuthenticationService authenticationService, final Predicate<RealmRepresentation> realmPredicate, final Predicate<UserRepresentation> userPredicate)
+            throws Exception {
         return withKeycloak(authenticationService, keycloak -> {
 
             List<RealmRepresentation> realmReps = keycloak.realms().findAll();
@@ -566,12 +598,12 @@ public class KeycloakUserApi implements UserApi, RealmApi {
                     String realm = realmRep.getRealm();
 
                     keycloak.realm(realm).users().list()
-                                    .stream()
-                                    .filter(userPredicate)
-                                    .forEachOrdered(userRep -> {
-                                        List<GroupRepresentation> groupReps = keycloak.realm(realm).users().get(userRep.getId()).groups();
-                                        userList.getItems().add(buildUser(userRep, groupReps));
-                                    });
+                            .stream()
+                            .filter(userPredicate)
+                            .forEachOrdered(userRep -> {
+                                List<GroupRepresentation> groupReps = keycloak.realm(realm).users().get(userRep.getId()).groups();
+                                userList.getItems().add(buildUser(userRep, groupReps));
+                            });
                 }
             }
 
@@ -585,8 +617,8 @@ public class KeycloakUserApi implements UserApi, RealmApi {
     @Override
     public UserList listAllUsers(AuthenticationService authenticationService) throws Exception {
         return queryUsers(authenticationService,
-                        realm -> getAttribute(realm, "namespace").isPresent(),
-                        user -> true);
+                realm -> getAttribute(realm, "namespace").isPresent(),
+                user -> true);
     }
 
     /**
@@ -595,8 +627,8 @@ public class KeycloakUserApi implements UserApi, RealmApi {
     @Override
     public UserList listUsers(AuthenticationService authenticationService, final String namespace) throws Exception {
         return queryUsers(authenticationService,
-                        realm -> hasAttribute(realm, "namespace", namespace),
-                        user -> true);
+                realm -> hasAttribute(realm, "namespace", namespace),
+                user -> true);
     }
 
     static Optional<String> getAttribute(final RealmRepresentation realm, final String attributeName) {
@@ -716,7 +748,7 @@ public class KeycloakUserApi implements UserApi, RealmApi {
                 .build();
     }
 
-    private static boolean matchesLabels (final UserRepresentation userRep, final Map<String,String> labels) {
+    private static boolean matchesLabels(final UserRepresentation userRep, final Map<String, String> labels) {
         for (final Map.Entry<String, String> label : labels.entrySet()) {
             if (userRep.getAttributes().get(label.getKey()) == null || !label.getValue().equals(userRep.getAttributes().get(label.getKey()).get(0))) {
                 return false;
@@ -728,15 +760,15 @@ public class KeycloakUserApi implements UserApi, RealmApi {
     @Override
     public UserList listUsersWithLabels(AuthenticationService authenticationService, String namespace, Map<String, String> labels) throws Exception {
         return queryUsers(authenticationService,
-                        realm -> hasAttribute(realm, "namespace", namespace),
-                        user -> matchesLabels(user, labels));
+                realm -> hasAttribute(realm, "namespace", namespace),
+                user -> matchesLabels(user, labels));
     }
 
     @Override
     public UserList listAllUsersWithLabels(AuthenticationService authenticationService, final Map<String, String> labels) throws Exception {
         return queryUsers(authenticationService,
-                        realm -> getAttribute(realm, "namespace").isPresent(),
-                        user -> matchesLabels(user, labels));
+                realm -> getAttribute(realm, "namespace").isPresent(),
+                user -> matchesLabels(user, labels));
     }
 
     @Override
@@ -799,4 +831,36 @@ public class KeycloakUserApi implements UserApi, RealmApi {
     public static String encodePart(final String part) {
         return URLEncoder.encode(part, StandardCharsets.UTF_8);
     }
+
+    private static boolean filterUserAnnotation(final Map.Entry<String, String> entry) {
+
+        // first check for the value length ...
+
+        if (entry.getValue() != null && entry.getValue().length() > MAX_ANNOTATION_VALUE_LENGTH) {
+            // ... value is too long, and will cause a problem
+            log.error("Found annotation value that exceeds limit - limit: {}, actualLengt: {}, key: {}, value: {}", MAX_ANNOTATION_VALUE_LENGTH, entry.getValue().length(),
+                    entry.getKey(), entry.getValue());
+            return false;
+        }
+
+        // next check the two lists ...
+
+        final String key = entry.getKey();
+
+        for (final Pattern pattern : ANNOTATION_BLACKLIST) {
+            if (pattern.matcher(key).matches()) {
+                return false;
+            }
+        }
+
+        for (final Pattern pattern : ANNOTATION_WHITELIST) {
+            if (pattern.matcher(key).matches()) {
+                return true;
+            }
+        }
+
+        // ... nothing left, reject!
+
+        return false;
+    }
 }
diff --git a/pkg/apis/iot/v1alpha1/funcs.go b/pkg/apis/iot/v1alpha1/funcs.go
index a3b14f4bd..89b862f0b 100644
--- a/pkg/apis/iot/v1alpha1/funcs.go
+++ b/pkg/apis/iot/v1alpha1/funcs.go
@@ -120,6 +120,26 @@ func (p *IoTProjectStatus) GetProjectCondition(t ProjectConditionType) *ProjectC
 	return &p.Conditions[len(p.Conditions)-1]
 }
 
+func (config *IoTConfigStatus) GetConfigCondition(t ConfigConditionType) *ConfigCondition {
+	for i, c := range config.Conditions {
+		if c.Type == t {
+			return &config.Conditions[i]
+		}
+	}
+
+	nc := ConfigCondition{
+		Type: t,
+		CommonCondition: CommonCondition{
+			Status:             corev1.ConditionUnknown,
+			LastTransitionTime: metav1.Now(),
+		},
+	}
+
+	config.Conditions = append(config.Conditions, nc)
+
+	return &config.Conditions[len(config.Conditions)-1]
+}
+
 func (c *CommonCondition) SetStatus(status corev1.ConditionStatus, reason string, message string) {
 
 	if c.Status != status {
diff --git a/pkg/apis/iot/v1alpha1/types_config.go b/pkg/apis/iot/v1alpha1/types_config.go
index 842c95473..49dce785b 100644
--- a/pkg/apis/iot/v1alpha1/types_config.go
+++ b/pkg/apis/iot/v1alpha1/types_config.go
@@ -199,11 +199,25 @@ type MqttAdapterConfig struct {
 
 type IoTConfigStatus struct {
 	Initialized bool   `json:"initialized"`
-	State       string `json:"state"`
+	Phase       string `json:"phase"`
+	PhaseReason string `json:"phaseReason,omitempty"`
 
 	AuthenticationServicePSK *string                  `json:"authenticationServicePSK"`
 	Adapters                 map[string]AdapterStatus `json:"adapters,omitempty"`
 	Services                 map[string]ServiceStatus `json:"services,omitempty"`
+
+	Conditions []ConfigCondition `json:"conditions"`
+}
+
+type ConfigConditionType string
+
+const (
+	ConfigConditionTypeReady ConfigConditionType = "Ready"
+)
+
+type ConfigCondition struct {
+	Type            ConfigConditionType `json:"type"`
+	CommonCondition `json:",inline"`
 }
 
 type CommonStatus struct {
@@ -227,7 +241,7 @@ type EndpointStatus struct {
 
 const (
 	ConfigStateWrongName = "WrongName"
-	ConfigStateRunning   = "Running"
+	ConfigStateReady     = "Ready"
 	ConfigStateFailed    = "Failed"
 )
 
diff --git a/pkg/apis/iot/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/iot/v1alpha1/zz_generated.deepcopy.go
index c1d087dbd..2c547d47e 100644
--- a/pkg/apis/iot/v1alpha1/zz_generated.deepcopy.go
+++ b/pkg/apis/iot/v1alpha1/zz_generated.deepcopy.go
@@ -316,6 +316,23 @@ func (in *CommonStatus) DeepCopy() *CommonStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ConfigCondition) DeepCopyInto(out *ConfigCondition) {
+	*out = *in
+	in.CommonCondition.DeepCopyInto(&out.CommonCondition)
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConfigCondition.
+func (in *ConfigCondition) DeepCopy() *ConfigCondition {
+	if in == nil {
+		return nil
+	}
+	out := new(ConfigCondition)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ConnectionInformation) DeepCopyInto(out *ConnectionInformation) {
 	*out = *in
@@ -727,6 +744,13 @@ func (in *IoTConfigStatus) DeepCopyInto(out *IoTConfigStatus) {
 			(*out)[key] = val
 		}
 	}
+	if in.Conditions != nil {
+		in, out := &in.Conditions, &out.Conditions
+		*out = make([]ConfigCondition, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
 	return
 }
 
diff --git a/pkg/controller/iotconfig/iotconfig_controller.go b/pkg/controller/iotconfig/iotconfig_controller.go
index 6a91368d6..7f833d75e 100644
--- a/pkg/controller/iotconfig/iotconfig_controller.go
+++ b/pkg/controller/iotconfig/iotconfig_controller.go
@@ -216,6 +216,11 @@ func (r *ReconcileIoTConfig) Reconcile(request reconcile.Request) (reconcile.Res
 	return r.updateFinalStatus(ctx, config, rc)
 }
 
+func syncConfigCondition(status *iotv1alpha1.IoTConfigStatus) {
+	ready := status.GetConfigCondition(iotv1alpha1.ConfigConditionTypeReady)
+	ready.SetStatusOkOrElse(status.Initialized, "NotReady", "infrastructure is not ready yet")
+}
+
 func (r *ReconcileIoTConfig) updateStatus(ctx context.Context, config *iotv1alpha1.IoTConfig, err error) error {
 
 	// we are initialized when there is no error
@@ -223,11 +228,13 @@ func (r *ReconcileIoTConfig) updateStatus(ctx context.Context, config *iotv1alph
 	config.Status.Initialized = err == nil
 
 	if config.Status.Initialized {
-		config.Status.State = iotv1alpha1.ConfigStateRunning
+		config.Status.Phase = iotv1alpha1.ConfigStateReady
 	} else {
-		config.Status.State = iotv1alpha1.ConfigStateFailed
+		config.Status.Phase = iotv1alpha1.ConfigStateFailed
 	}
 
+	syncConfigCondition(&config.Status)
+
 	return r.client.Status().Update(ctx, config)
 }
 
@@ -247,9 +254,11 @@ func (r *ReconcileIoTConfig) updateFinalStatus(ctx context.Context, config *iotv
 func (r *ReconcileIoTConfig) failWrongConfigName(ctx context.Context, config *iotv1alpha1.IoTConfig) (reconcile.Result, error) {
 
 	config.Status.Initialized = false
-	config.Status.State = iotv1alpha1.ConfigStateWrongName
+	config.Status.Phase = iotv1alpha1.ConfigStateWrongName
+
+	syncConfigCondition(&config.Status)
 
-	if err := r.client.Update(ctx, config); err != nil {
+	if err := r.client.Status().Update(ctx, config); err != nil {
 		return reconcile.Result{}, err
 	}
 
diff --git a/pkg/util/password.go b/pkg/util/password.go
index e3cb8362b..51305a3c4 100644
--- a/pkg/util/password.go
+++ b/pkg/util/password.go
@@ -11,7 +11,7 @@ import (
 )
 
 var (
-	PossibleCharacters = []rune("abcdefghijklmnopqrstuvwxyz" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "0123456789" + ";:_,.-#+*=?()/&%$!")
+	PossibleCharacters = []rune("abcdefghijklmnopqrstuvwxyz" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "0123456789" + ";:,.-_=")
 	MaxLen             = big.NewInt(int64(len(PossibleCharacters)))
 )
 
diff --git a/pom.xml b/pom.xml
index 94c030038..6c425fe18 100644
--- a/pom.xml
+++ b/pom.xml
@@ -91,7 +91,7 @@
         <properties-maven-plugin.version>1.0.0</properties-maven-plugin.version>
         <maven-resources-plugin.version>3.1.0</maven-resources-plugin.version>
         <maven-checkstyle-plugin.version>3.0.0</maven-checkstyle-plugin.version>
-        <maven-compiler-plugin.version>3.8.1</maven-compiler-plugin.version>
+        <maven-compiler-plugin.version>3.8.0</maven-compiler-plugin.version>
         <maven-jar-plugin.version>3.1.1</maven-jar-plugin.version>
         <maven-shade-plugin.version>3.1.0</maven-shade-plugin.version>
         <maven-surefire-plugin.version>2.22.1</maven-surefire-plugin.version>
@@ -460,6 +460,11 @@
                 <artifactId>jackson-databind</artifactId>
                 <version>${jackson-databind.version}</version>
             </dependency>
+            <dependency>
+                <groupId>com.fasterxml.jackson.datatype</groupId>
+                <artifactId>jackson-datatype-jsr310</artifactId>
+                <version>${jackson-databind.version}</version>
+            </dependency>
             <dependency>
                 <groupId>com.fasterxml.jackson.module</groupId>
                 <artifactId>jackson-module-jsonSchema</artifactId>
@@ -667,9 +672,6 @@
                     <groupId>org.apache.maven.plugins</groupId>
                     <artifactId>maven-compiler-plugin</artifactId>
                     <version>${maven-compiler-plugin.version}</version>
-                    <configuration>
-                       <forceJavacCompilerUse>true</forceJavacCompilerUse>
-                    </configuration>
                 </plugin>
                 <plugin>
                     <groupId>org.apache.maven.plugins</groupId>
diff --git a/templates/crds/010-iotconfigs.crd.yaml b/templates/crds/010-iotconfigs.crd.yaml
index 34339fc2f..f1689c25f 100644
--- a/templates/crds/010-iotconfigs.crd.yaml
+++ b/templates/crds/010-iotconfigs.crd.yaml
@@ -14,5 +14,10 @@ spec:
     singular: iotconfig
     shortNames:
     - icfg
+  additionalPrinterColumns:
+  - name: Phase
+    type: string
+    description: Phase of the IoT config
+    JSONPath: .status.phase
   subresources:
     status: {}
diff --git a/templates/iot/examples/infinispan/common/010-Config.yaml b/templates/iot/examples/infinispan/common/010-Config.yaml
index a7b8701bb..18f49db98 100644
--- a/templates/iot/examples/infinispan/common/010-Config.yaml
+++ b/templates/iot/examples/infinispan/common/010-Config.yaml
@@ -271,7 +271,9 @@ data:
               <subsystem xmlns="urn:infinispan:server:core:9.4" default-cache-container="clustered">
                   <cache-container name="clustered" default-cache="default" statistics="true">
                       <transport lock-timeout="60000"/>
-                      <global-state/>
+                      <global-state
+                            enable="true"
+                            persistent-location="global-data"/>
                       <distributed-cache-configuration name="transactional">
                           <transaction mode="NON_XA" locking="PESSIMISTIC"/>
                       </distributed-cache-configuration>
@@ -322,10 +324,12 @@ data:
                         
                         <replicated-cache name="LuceneIndexesMetadata">
                             <indexing index="NONE" />
+                            <rocksdb-store shared="false" fetch-state="true" passivation="false" preload="true" />
                         </replicated-cache>
                         
                         <distributed-cache name="LuceneIndexesData">
                             <indexing index="NONE" />
+                            <rocksdb-store shared="false" fetch-state="true" passivation="false" preload="true" />
                         </distributed-cache>
 
                        <distributed-cache owners="3" mode="SYNC" remote-timeout="15000" name="devices">
@@ -335,6 +339,7 @@ data:
                                    org.infinispan.query.indexmanager.InfinispanIndexManager
                                </property>
                            </indexing>
+                          <rocksdb-store shared="false" fetch-state="true" passivation="false" preload="true" />
                        </distributed-cache>
 
                        <distributed-cache owners="3" mode="SYNC" remote-timeout="15000" name="adapterCredentials">
diff --git a/templates/iot/examples/infinispan/manual/050-StatefulSet.yaml b/templates/iot/examples/infinispan/manual/050-StatefulSet.yaml
index 1d4f0155c..2998252f2 100644
--- a/templates/iot/examples/infinispan/manual/050-StatefulSet.yaml
+++ b/templates/iot/examples/infinispan/manual/050-StatefulSet.yaml
@@ -91,9 +91,20 @@ spec:
         volumeMounts:
         - mountPath: /opt/jboss/infinispan-server/standalone/configuration/custom
           name: infinispan-configuration
+        - mountPath: /opt/jboss/infinispan-server/standalone/data
+          name: infinispan-data
       terminationGracePeriodSeconds: 120
       volumes:
       - name: infinispan-configuration
         configMap:
           defaultMode: 420
           name: infinispan-configuration
+  volumeClaimTemplates:
+  - metadata:
+      name: infinispan-data
+    spec:
+      accessModes:
+      - ReadWriteOnce
+      resources:
+        requests:
+          storage: 32Gi
\ No newline at end of file
diff --git a/templates/iot/examples/iot-config-dev.yaml b/templates/iot/examples/iot-config-dev.yaml
index 4a38de13e..138978a83 100644
--- a/templates/iot/examples/iot-config-dev.yaml
+++ b/templates/iot/examples/iot-config-dev.yaml
@@ -8,7 +8,7 @@ spec:
       infinispan:
         server:
           external:
-            host: infinispan-server
+            host: infinispan
             port: 11222
             username: app
             password: test12
diff --git a/templates/iot/examples/iot-config-k8s.yaml b/templates/iot/examples/iot-config-k8s.yaml
index 402ca5c33..c65e83608 100644
--- a/templates/iot/examples/iot-config-k8s.yaml
+++ b/templates/iot/examples/iot-config-k8s.yaml
@@ -15,7 +15,7 @@ spec:
       infinispan:
         server:
           external:
-            host: infinispan-server
+            host: infinispan
             port: 11222
             username: app
             password: test12
diff --git a/templates/iot/examples/iot-config.yaml b/templates/iot/examples/iot-config.yaml
index 050e6ce2c..dded373b1 100644
--- a/templates/iot/examples/iot-config.yaml
+++ b/templates/iot/examples/iot-config.yaml
@@ -8,7 +8,7 @@ spec:
       infinispan:
         server:
           external:
-            host: infinispan-server
+            host: infinispan
             port: 11222
             username: app
             password: test12


Test patch:
diff --git a/api-model/src/test/java/io/enmasse/iot/model/v1/IoTConfigTest.java b/api-model/src/test/java/io/enmasse/iot/model/v1/IoTConfigTest.java
index af9faebe0..ef6e88af3 100644
--- a/api-model/src/test/java/io/enmasse/iot/model/v1/IoTConfigTest.java
+++ b/api-model/src/test/java/io/enmasse/iot/model/v1/IoTConfigTest.java
@@ -89,7 +89,7 @@ public class IoTConfigTest {
 
         assertThat(value.getStatus(), notNullValue());
         assertThat(value.getStatus().isInitialized(), is(true));
-        assertThat(value.getStatus().getState(), is("ok"));
+        assertThat(value.getStatus().getPhase(), is("Ready"));
         assertThat(value.getStatus().getAuthenticationServicePSK(), is("psk"));
 
 
diff --git a/api-model/src/test/resources/io/enmasse/iot/model/v1/resources/iot-config.json b/api-model/src/test/resources/io/enmasse/iot/model/v1/resources/iot-config.json
index 529a90189..77dd44ef9 100644
--- a/api-model/src/test/resources/io/enmasse/iot/model/v1/resources/iot-config.json
+++ b/api-model/src/test/resources/io/enmasse/iot/model/v1/resources/iot-config.json
@@ -37,7 +37,14 @@
 	},
 	"status": {
 		"initialized": true,
-		"state": "ok",
-		"authenticationServicePSK": "psk"
+		"phase": "Ready",
+		"authenticationServicePSK": "psk",
+		"conditions": [
+			{
+				"lastTransitionTime": "2019-10-07T11:37:57Z",
+				"status": "True",
+				"type": "Ready"
+			}
+		]
 	}
 }
\ No newline at end of file
diff --git a/keycloak-user-api/src/test/java/io/enmasse/user/keycloak/KeycloakUserApiTest.java b/keycloak-user-api/src/test/java/io/enmasse/user/keycloak/KeycloakUserApiTest.java
index d05def7b7..93dd1ab4a 100644
--- a/keycloak-user-api/src/test/java/io/enmasse/user/keycloak/KeycloakUserApiTest.java
+++ b/keycloak-user-api/src/test/java/io/enmasse/user/keycloak/KeycloakUserApiTest.java
@@ -356,14 +356,38 @@ public class KeycloakUserApiTest {
     @Test
     public void testAnnotations() {
         final Map<String,String> annotations = new HashMap<>();
-        annotations.put("foo", "bar");
-        annotations.put("bar", "baz");
+        annotations.put("enmasse.io/foo", "bar");
+        annotations.put("iot.enmasse.io/bar", "baz");
 
         final List<String> encoded = KeycloakUserApi.annotationsToString(annotations);
         assertNotNull(encoded);
-        assertEquals(2,encoded.size());
 
         final Map<String, String> actualAnnotations = KeycloakUserApi.annotationsFromString(encoded);
         assertEquals(annotations, actualAnnotations);
     }
+
+    @Test
+    public void testDiscardAnnotation() {
+        final Map<String,String> discardedAnnotations = new HashMap<>();
+        discardedAnnotations.put("enmasse.io/too-long", chars(256,'a'));
+        discardedAnnotations.put("foo", "wrong-pattern");
+
+        final Map<String,String> acceptedAnnotations = new HashMap<>();
+        acceptedAnnotations.put("enmasse.io/just-right", chars(255,'a'));
+
+        Map<String,String> annotations = new HashMap<>(discardedAnnotations);
+        annotations.putAll(acceptedAnnotations);
+
+        final List<String> encoded = KeycloakUserApi.annotationsToString(annotations);
+        assertNotNull(encoded);
+
+        final Map<String, String> actualAnnotations = KeycloakUserApi.annotationsFromString(encoded);
+        assertEquals(acceptedAnnotations, actualAnnotations);
+    }
+
+    private static String chars(int amount, char c) {
+        final char result[] = new char[amount];
+        Arrays.fill(result, c);
+        return String.valueOf(result);
+    }
 }
diff --git a/systemtests/README.md b/systemtests/README.md
index 8dec76fd5..3a9f69e32 100644
--- a/systemtests/README.md
+++ b/systemtests/README.md
@@ -25,5 +25,23 @@ the [documentation](http://enmasse.io/documentation/master/openshift/#quickstart
 
 Where $PROFILE can be:
 * systemtests
-* systemtests-marathon
-* systemtests-upgrade
+* marathon
+* iot
+* shared
+* isolated
+* shared-iot
+* isolated-iot
+* smoke
+* smoke-iot
+* upgrade
+
+## Running upgrade test
+
+    mkdir templates/build -pv
+    export START_VERSION=0.28.2
+    wget https://github.com/EnMasseProject/enmasse/releases/download/${START_VERSION}/enmasse-${START_VERSION}.tgz -O templates/build/enmasse-${START_VERSION}.tgz
+    tar zxvf templates/build/enmasse-${version}.tgz -C templates/build
+    make templates
+    export START_TEMPLATES=${pwd}/templates/build/enmasse-${START_VERSION}
+    export UPGRADE_TEMPLATES=${pwd}/templates/build/enmasse-latest
+    make SYSTEMTEST_PROFILE=upgrade
diff --git a/systemtests/src/main/java/io/enmasse/systemtest/utils/IoTUtils.java b/systemtests/src/main/java/io/enmasse/systemtest/utils/IoTUtils.java
index e09aa3dfd..09252da43 100644
--- a/systemtests/src/main/java/io/enmasse/systemtest/utils/IoTUtils.java
+++ b/systemtests/src/main/java/io/enmasse/systemtest/utils/IoTUtils.java
@@ -74,7 +74,7 @@ public class IoTUtils {
             }
         }
         if (!isReady) {
-            String jsonStatus = config != null && config.getStatus() != null ? config.getStatus().getState() : "";
+            String jsonStatus = config != null && config.getStatus() != null ? config.getStatus().getPhase() : "";
             throw new IllegalStateException("IoTConfig " + Objects.requireNonNull(config).getMetadata().getName() + " is not in Ready state within timeout: " + jsonStatus);
         }
 
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/iot/isolated/SimpleK8sDeployTest.java b/systemtests/src/test/java/io/enmasse/systemtest/iot/isolated/SimpleK8sDeployTest.java
index fcb4c46b5..c0190115f 100644
--- a/systemtests/src/test/java/io/enmasse/systemtest/iot/isolated/SimpleK8sDeployTest.java
+++ b/systemtests/src/test/java/io/enmasse/systemtest/iot/isolated/SimpleK8sDeployTest.java
@@ -13,7 +13,6 @@ import io.enmasse.iot.model.v1.IoTConfigBuilder;
 import io.enmasse.systemtest.Environment;
 import io.enmasse.systemtest.bases.TestBase;
 import io.enmasse.systemtest.bases.iot.ITestIoTIsolated;
-import io.enmasse.systemtest.bases.iot.ITestIoTShared;
 import io.enmasse.systemtest.cmdclients.KubeCMDClient;
 import io.enmasse.systemtest.platform.Kubernetes;
 import io.enmasse.systemtest.utils.IoTUtils;
@@ -21,7 +20,6 @@ import io.enmasse.systemtest.utils.TestUtils;
 import io.fabric8.kubernetes.api.model.Quantity;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/iot/isolated/registry/DeviceRegistryTest.java b/systemtests/src/test/java/io/enmasse/systemtest/iot/isolated/registry/DeviceRegistryTest.java
index 71a76ad8e..fbe12b887 100644
--- a/systemtests/src/test/java/io/enmasse/systemtest/iot/isolated/registry/DeviceRegistryTest.java
+++ b/systemtests/src/test/java/io/enmasse/systemtest/iot/isolated/registry/DeviceRegistryTest.java
@@ -19,11 +19,14 @@ import io.enmasse.systemtest.utils.IoTUtils;
 import org.eclipse.hono.service.management.device.Device;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Tag;
+
 import java.time.Duration;
 import java.time.Instant;
 import java.util.UUID;
 
 import static io.enmasse.systemtest.TestTag.SMOKE;
+import static java.net.HttpURLConnection.HTTP_CONFLICT;
+import static java.net.HttpURLConnection.HTTP_CREATED;
 import static java.net.HttpURLConnection.HTTP_NOT_FOUND;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -37,11 +40,11 @@ abstract class DeviceRegistryTest extends TestBase implements ITestIoTIsolated {
 
 
     private String randomDeviceId;
-    private IoTConfig iotConfig = null;
-    private IoTProject iotProject = null;
-    private Endpoint deviceRegistryEndpoint = null;
-    private Endpoint httpAdapterEndpoint = null;
-    private DeviceRegistryClient client = null;
+    private IoTConfig iotConfig;
+    private IoTProject iotProject;
+    private Endpoint deviceRegistryEndpoint;
+    private Endpoint httpAdapterEndpoint;
+    private DeviceRegistryClient client;
 
     private UserCredentials credentials;
 
@@ -217,5 +220,19 @@ abstract class DeviceRegistryTest extends TestBase implements ITestIoTIsolated {
         assertEquals(HTTP_NOT_FOUND, response.statusCode());
     }
 
+    protected void doCreateDuplicateDeviceFails() throws Exception {
+        var tenantId = isolatedIoTManager.getTenantId();
+        var deviceId = UUID.randomUUID().toString();
+
+        // create device
+
+        var response = client.registerDeviceWithResponse(tenantId, deviceId);
+        assertEquals(HTTP_CREATED, response.statusCode());
+
+        // create device a second time
+
+        var response2 = client.registerDeviceWithResponse(tenantId, deviceId);
+        assertEquals(HTTP_CONFLICT, response2.statusCode());
+    }
 
 }
\ No newline at end of file
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/iot/isolated/registry/FileDeviceRegistryTest.java b/systemtests/src/test/java/io/enmasse/systemtest/iot/isolated/registry/FileDeviceRegistryTest.java
index 614bc80bb..c5288e09c 100644
--- a/systemtests/src/test/java/io/enmasse/systemtest/iot/isolated/registry/FileDeviceRegistryTest.java
+++ b/systemtests/src/test/java/io/enmasse/systemtest/iot/isolated/registry/FileDeviceRegistryTest.java
@@ -5,11 +5,9 @@
 package io.enmasse.systemtest.iot.isolated.registry;
 
 import io.enmasse.iot.model.v1.IoTConfigBuilder;
-import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 
 import static io.enmasse.systemtest.iot.DefaultDeviceRegistry.newFileBased;
-import static io.enmasse.systemtest.iot.DefaultDeviceRegistry.newInfinispanBased;
 import static io.enmasse.systemtest.utils.IoTUtils.assertCorrectRegistryType;
 
 class FileDeviceRegistryTest extends DeviceRegistryTest {
@@ -48,20 +46,9 @@ class FileDeviceRegistryTest extends DeviceRegistryTest {
         super.doTestDeviceCredentials();
     }
 
-    @Test
-    @Disabled("Caches expire a bit unpredictably")
-    void testCacheExpiryForCredentials() throws Exception {
-        super.doTestCacheExpiryForCredentials();
-    }
-
     @Test
     void testSetExpiryForCredentials() throws Exception {
         super.doTestSetExpiryForCredentials();
     }
 
-    @Test
-    @Disabled
-    void testCreateForNonExistingTenantFails() throws Exception {
-        super.doTestCreateForNonExistingTenantFails();
-    }
 }
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/iot/isolated/registry/InfinispanDeviceRegistryTest.java b/systemtests/src/test/java/io/enmasse/systemtest/iot/isolated/registry/InfinispanDeviceRegistryTest.java
index 4ce42fd3d..8227fc287 100644
--- a/systemtests/src/test/java/io/enmasse/systemtest/iot/isolated/registry/InfinispanDeviceRegistryTest.java
+++ b/systemtests/src/test/java/io/enmasse/systemtest/iot/isolated/registry/InfinispanDeviceRegistryTest.java
@@ -62,4 +62,9 @@ class InfinispanDeviceRegistryTest extends DeviceRegistryTest {
     void testCreateForNonExistingTenantFails() throws Exception {
         super.doTestCreateForNonExistingTenantFails();
     }
+
+    @Test
+    void testCreateDuplicateDeviceFails() throws Exception {
+        super.doCreateDuplicateDeviceFails();
+    }
 }
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/isolated/CommonTest.java b/systemtests/src/test/java/io/enmasse/systemtest/isolated/CommonTest.java
index 3fce04003..fc79bb97b 100644
--- a/systemtests/src/test/java/io/enmasse/systemtest/isolated/CommonTest.java
+++ b/systemtests/src/test/java/io/enmasse/systemtest/isolated/CommonTest.java
@@ -29,16 +29,14 @@ import org.slf4j.Logger;
 import java.util.AbstractMap;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import static io.enmasse.systemtest.TestTag.ISOLATED;
 import static io.enmasse.systemtest.TestTag.NON_PR;
@@ -416,16 +414,21 @@ class CommonTest extends TestBase implements ITestBaseIsolated {
             for (Address addr : brokeredAddresses) {
                 log.info("Starting messaging in address {} and address space {}", addr.getSpec().getAddress(), brokered.getMetadata().getName());
                 for (Label label : labels) {
+                    getClientUtils().assertCanConnect(brokered, user, brokeredAddresses, resourcesManager);
                     doMessagingDuringRestart(label, runningPodsBefore, user, brokered, addr);
+                    getClientUtils().assertCanConnect(brokered, user, brokeredAddresses, resourcesManager);
                 }
             }
 
             for (Address addr : standardAddresses) {
                 log.info("Starting messaging in address {} and address space {}", addr.getSpec().getAddress(), standard.getMetadata().getName());
                 for (Label label : labels) {
+                    getClientUtils().assertCanConnect(standard, user, standardAddresses, resourcesManager);
                     doMessagingDuringRestart(label, runningPodsBefore, user, standard, addr);
+                    getClientUtils().assertCanConnect(standard, user, standardAddresses, resourcesManager);
                 }
             }
+
         } finally {
             // Ensure that EnMasse's API services are finished re-registering (after api-server restart) before ending
             // the test otherwise test clean-up will fail.
@@ -438,13 +441,6 @@ class CommonTest extends TestBase implements ITestBaseIsolated {
     // help methods
     /////////////////////////////////////////////////////////////////////
 
-    private void sendMessage(AmqpClient client, String address, AtomicInteger counter) throws Exception {
-        counter.incrementAndGet();
-        Future<Integer> sent = client.sendMessages(address, Collections.singletonList(UUID.randomUUID().toString()));
-        log.info("Message sent");
-        assertEquals(1, sent.get(15, TimeUnit.SECONDS));
-    }
-
     private void assertSystemWorks(AddressSpace brokered, AddressSpace standard, UserCredentials existingUser,
                                    List<Address> brAddresses, List<Address> stAddresses) throws Exception {
         log.info("Check if system works");
@@ -456,59 +452,50 @@ class CommonTest extends TestBase implements ITestBaseIsolated {
         resourcesManager.createOrUpdateUser(standard, new UserCredentials("jura", "fura"));
     }
 
-    private void doMessagingDuringRestart(Label label, int runningPodsBefore, UserCredentials user, AddressSpace brokered, Address addr) throws Exception {
+    private void doMessagingDuringRestart(Label label, int runningPodsBefore, UserCredentials user, AddressSpace space, Address addr) throws Exception {
         log.info("Starting messaging");
         AddressType addressType = AddressType.getEnum(addr.getSpec().getType());
-        AmqpClient client = getAmqpClientFactory().createAddressClient(brokered, addressType);
+        AmqpClient client = getAmqpClientFactory().createAddressClient(space, addressType);
         client.getConnectOptions().setCredentials(user);
-        AtomicInteger counter = new AtomicInteger(0);
-        CompletableFuture<Object> future = doConcurrentMessaging(client, addr.getSpec().getAddress(), counter);
-        log.info("Restarting {}", label.labelValue);
-        KubeCMDClient.deletePodByLabel(label.getLabelName(), label.getLabelValue());
-        TestUtils.waitForExpectedReadyPods(kubernetes, kubernetes.getInfraNamespace(), runningPodsBefore, new TimeoutBudget(10, TimeUnit.MINUTES));
-        if (future.isCompletedExceptionally()) {
-            future.get();
-        }
-        future.complete(new Object());
-        assertTrue(counter.get() > 1, "receive messages did not work");
-    }
 
-    private CompletableFuture<Object> doConcurrentMessaging(AmqpClient client, String address, AtomicInteger counter) {
+        var restart = new CompletableFuture<Object>();
 
-        CompletableFuture<Object> resultPromise = new CompletableFuture<>();
+        var recvFut = client.recvMessages(addr.getSpec().getAddress(), msg -> {
+            log.info("Message received");
+            if (restart.isDone()) {
+                return true;
+            }
+            return false;
+        });
 
-        CompletableFuture.runAsync(() -> {
-            client.recvMessages(address, msg -> {
-                log.info("Message received");
-                if (resultPromise.isDone()) {
-                    return true;
-                }
-                try {
-                    Thread.sleep(500);
-                } catch (InterruptedException e) {
-                    log.error("Error waiting between sends", e);
-                    resultPromise.completeExceptionally(e);
-                    return true;
-                }
-                CompletableFuture.runAsync(() -> {
+
+        var sendFut = client.sendMessages(addr.getSpec().getAddress(),
+                Stream.generate(() -> UUID.randomUUID().toString()).limit(1000).collect(Collectors.toList()),
+                msg -> {
+                    if (restart.isDone()) {
+                        return true;
+                    }
                     try {
-                        sendMessage(client, address, counter);
-                    } catch (Exception e) {
-                        log.error("Error sending message", e);
-                        resultPromise.completeExceptionally(e);
+                        Thread.sleep(500);
+                    } catch (InterruptedException e) {
+                        log.error("Error waiting between sends", e);
+                        restart.completeExceptionally(e);
+                        return true;
                     }
-                }, runnable -> new Thread(runnable).start());
-                return false;
-            });
-            try {
-                sendMessage(client, address, counter);
-            } catch (Exception e) {
-                log.error("Error sending message", e);
-                resultPromise.completeExceptionally(e);
-            }
-        }, runnable -> new Thread(runnable).start());
+                    return false;
+                });
+
+        log.info("Restarting {}", label.labelValue);
+        KubeCMDClient.deletePodByLabel(label.getLabelName(), label.getLabelValue());
+        TestUtils.waitForExpectedReadyPods(kubernetes, kubernetes.getInfraNamespace(), runningPodsBefore, new TimeoutBudget(10, TimeUnit.MINUTES));
+        if (restart.isCompletedExceptionally()) {
+            restart.get();
+        }
+        restart.complete(new Object());
 
-        return resultPromise;
+        int received = recvFut.get(10, TimeUnit.SECONDS).size();
+        int sent = sendFut.get(10, TimeUnit.SECONDS);
+        assertEquals(sent, received, "Missmatch between messages sent and received");
     }
 
     private class Label {
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/isolated/upgrade/UpgradeTest.java b/systemtests/src/test/java/io/enmasse/systemtest/isolated/upgrade/UpgradeTest.java
index 9a5cbbaaf..e9e2edda0 100644
--- a/systemtests/src/test/java/io/enmasse/systemtest/isolated/upgrade/UpgradeTest.java
+++ b/systemtests/src/test/java/io/enmasse/systemtest/isolated/upgrade/UpgradeTest.java
@@ -252,9 +252,6 @@ class UpgradeTest extends TestBase implements ITestIsolatedStandard {
         String images = Files.readString(imageEnvDir);
         log.info("Expected images: {}", images);
 
-        kubernetes.listPods().stream().filter(pod -> pod.getMetadata().getName().matches("^qdrouterd-.*-0"))
-                .forEach(pod -> kubernetes.deletePod(kubernetes.getInfraNamespace(), pod.getMetadata().getName()));
-
         TestUtils.waitUntilCondition("Images are updated", (phase) -> {
             AtomicBoolean ready = new AtomicBoolean(true);
             log.info("=======================================================");
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/shared/brokered/api/UserApiTest.java b/systemtests/src/test/java/io/enmasse/systemtest/shared/brokered/api/UserApiTest.java
index 08a297aef..d6ee25d95 100644
--- a/systemtests/src/test/java/io/enmasse/systemtest/shared/brokered/api/UserApiTest.java
+++ b/systemtests/src/test/java/io/enmasse/systemtest/shared/brokered/api/UserApiTest.java
@@ -34,7 +34,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
 public class UserApiTest extends TestBase implements ITestSharedBrokered {
 
     private Map<AddressSpace, User> users = new HashMap<>();
-    private Logger LOGGER = CustomLogger.getLogger();
+    private static final Logger LOGGER = CustomLogger.getLogger();
 
     @AfterEach
     void cleanUsers() {
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/shared/standard/api/UserApiTest.java b/systemtests/src/test/java/io/enmasse/systemtest/shared/standard/api/UserApiTest.java
index b1c17d9b0..b3ef2ed0b 100644
--- a/systemtests/src/test/java/io/enmasse/systemtest/shared/standard/api/UserApiTest.java
+++ b/systemtests/src/test/java/io/enmasse/systemtest/shared/standard/api/UserApiTest.java
@@ -21,6 +21,7 @@ import io.enmasse.systemtest.utils.UserUtils;
 import io.enmasse.user.model.v1.DoneableUser;
 import io.enmasse.user.model.v1.Operation;
 import io.enmasse.user.model.v1.User;
+import io.enmasse.user.model.v1.UserAuthenticationType;
 import io.enmasse.user.model.v1.UserAuthorizationBuilder;
 import io.enmasse.user.model.v1.UserBuilder;
 import io.enmasse.user.model.v1.UserCrd;
@@ -146,45 +147,54 @@ public class UserApiTest extends TestBase implements ITestSharedStandard {
     void testSetAnnotations() {
 
         var client = Kubernetes.getInstance().getClient();
-        var userCrd = client.customResources(UserCrd.messagingUser(), User.class, UserList.class, DoneableUser.class);
+        var userCrd = client.customResources(UserCrd.messagingUser(), User.class, UserList.class, DoneableUser.class).inNamespace(getSharedAddressSpace().getMetadata().getNamespace());
 
         var name = getSharedAddressSpace().getMetadata().getName()+ ".foo";
         var user = new UserBuilder()
                 .withNewMetadata()
                 .withNamespace(getSharedAddressSpace().getMetadata().getNamespace())
                 .withName(name)
-                .addToAnnotations("annotation1", "value1")
-                .addToAnnotations("annotation2", "value2")
+                .addToAnnotations("enmasse.io/annotation1", "value1")
+                .addToAnnotations("enmasse.io/annotation2", "value2")
                 .endMetadata()
+
+                .withNewSpec()
+                .withUsername("foo")
+                .withNewAuthentication()
+                .withType(UserAuthenticationType.password)
+                .withPassword("foo")
+                .endAuthentication()
+                .endSpec()
+
                 .build();
 
         // create
         userCrd.create(user);
 
         // read again
-        var actualUser = userCrd.inNamespace(getSharedAddressSpace().getMetadata().getNamespace()).withName(name).get();
+        var actualUser = userCrd.withName(name).get();
 
         assertNotNull(actualUser);
-        assertThat(actualUser.getAnnotation("annotation1"), is("value1"));
-        assertThat(actualUser.getAnnotation("annotation2"), is("value2"));
+        assertThat(actualUser.getAnnotation("enmasse.io/annotation1"), is("value1"));
+        assertThat(actualUser.getAnnotation("enmasse.io/annotation2"), is("value2"));
 
         var updateUser = new UserBuilder(actualUser)
                 .editOrNewMetadata()
-                .addToAnnotations("annotation1", "value1a")
-                .removeFromAnnotations("annotation2")
-                .addToAnnotations("annotation3", "value3")
+                .addToAnnotations("enmasse.io/annotation1", "value1a")
+                .removeFromAnnotations("enmasse.io/annotation2")
+                .addToAnnotations("enmasse.io/annotation3", "value3")
                 .endMetadata()
                 .build();
 
         userCrd.createOrReplace(updateUser);
 
         // read again
-        actualUser = userCrd.inNamespace(getSharedAddressSpace().getMetadata().getNamespace()).withName(name).get();
+        actualUser = userCrd.withName(name).get();
 
         assertNotNull(actualUser);
-        assertThat(actualUser.getAnnotation("annotation1"), is("value1a"));
-        assertThat(actualUser.getAnnotation("annotation2"), nullValue());
-        assertThat(actualUser.getAnnotation("annotation3"), is("value3"));
+        assertThat(actualUser.getAnnotation("enmasse.io/annotation1"), is("value1a"));
+        assertThat(actualUser.getAnnotation("enmasse.io/annotation2"), nullValue());
+        assertThat(actualUser.getAnnotation("enmasse.io/annotation3"), is("value3"));
 
     }
 }