Repository: EnMasseProject/enmasse
Pull request #3256 (2019-10-01 11:05:56)

Fix upgrade test
### Type of change

_Select the type of your PR_

- Bugfix

### Description

This fixes an issue when upgrading where the router statefulset is not
completely rolled out after the upgrade. Once upgraded, the new agent
image will delete the old probe listener on the old router, causing the
readiness probe to fail. While the readiness probe is failing, the
rollout fails to proceed.

This fix re-adds the old probe which have to be kept around until
upgrading from 0.26.x is no longer supported.

Another upgrade issue fixed in this PR is that a missing sharded-queue/broker.xml causes sharded queues to not be reachable, as the draining broker is missing its broker.xml to be able to connect to the router.

### Checklist

_Please go through this checklist and make sure all applicable tasks have been done_

- [ ] Update/write design documentation in `./documentation/design`
- [ ] Write tests and make sure they pass
- [ ] Update documentation
- [ ] Check RBAC rights for Kubernetes / OpenShift roles
- [x] Try your changes from Pod inside your Kubernetes and OpenShift cluster, not just locally
- [ ] Reference relevant issue(s) and close them after merging
- [ ] Update CHANGELOG.md


lulf: @enmasse-ci run tests profile=upgrade
enmasse-ci: :heavy_exclamation_mark: **Build Failed** :heavy_exclamation_mark:
lulf: @enmasse-ci run tests profile=upgrade upgrade_from=0.26.5
enmasse-ci: :heavy_exclamation_mark: **Build Failed** :heavy_exclamation_mark:
k-wall: Good sleuthing @lulf.  There was a workaround added to `io.enmasse.systemtest.isolated.upgrade.UpgradeTest#checkImagesUpdated` and documented `documentation/modules/proc-upgrading-router-pods.adoc` which should be removed if this is fix is successful. @jenmalloy 
lulf: @enmasse-ci run tests profile=upgrade upgrade_from=0.26.5
enmasse-ci: ### :x: Test Summary :x:
**TEST_PROFILE**: upgrade
**TEST_CASE:** 
**TOTAL:** 2
**PASS:** 0
**FAIL:** 2
**SKIP:** 0
### :heavy_exclamation_mark: Test Failures :heavy_exclamation_mark:
- testUpgradeBundle in io.enmasse.systemtest.isolated.upgrade.UpgradeTest
- testUpgradeAnsible in io.enmasse.systemtest.isolated.upgrade.UpgradeTest
lulf: @enmasse-ci run tests profile=upgrade upgrade_from=0.26.5
lulf: @k-wall i put a README.md as well as a comment in broker.xml explaining it. Any ideas on a better place to put it?

Patch:
diff --git a/agent/lib/router_config.js b/agent/lib/router_config.js
index 502d067a7..a55b6f695 100644
--- a/agent/lib/router_config.js
+++ b/agent/lib/router_config.js
@@ -86,7 +86,22 @@ function listener_compare(a, b) {
 }
 
 function same_listener_definition(a, b) {
-    return a.host === b.host && a.port === b.port && a.authenticatePeer === b.authenticatePeer && a.metrics === b.metrics && a.healthz === b.healthz && a.http === b.http && a.websockets === b.websockets && a.httpRootDir === b.httpRootDir;
+    // NOTE: Special comparsion needed for legacy probe as schema for listener changed between
+    // router versions
+    if (a.port === '56711' && b.port === '56711') {
+        return a.host === b.host &&
+            a.port === b.port &&
+            a.authenticatePeer === b.authenticatePeer;
+    } else {
+        return a.host === b.host &&
+            a.port === b.port &&
+            a.authenticatePeer === b.authenticatePeer && 
+            a.metrics === b.metrics &&
+            a.healthz === b.healthz &&
+            a.http === b.http &&
+            a.websockets === b.websockets &&
+            a.httpRootDir === b.httpRootDir;
+    }
 }
 
 function listener_describe (a) {
@@ -390,6 +405,9 @@ function desired_address_config(high_level_address_definitions) {
         }
     }
     config.add_listener({host:'0.0.0.0', port: '8080', authenticatePeer: false, metrics: true, healthz: true, http: true, websockets: false, httpRootDir: 'invalid'})
+    // NOTE: The following listener exists in order to be able to proceed with upgrade from 0.26.x
+    // to master as the statefulset controller will not start rollout if probe is failing
+    config.add_listener({host:'0.0.0.0', port: '56711', authenticatePeer: true});
     sort_config(config);
     log.debug('mapped %j => %j', high_level_address_definitions, config);
     return config;
diff --git a/api-model/pom.xml b/api-model/pom.xml
index a9e31938c..ed99e522c 100644
--- a/api-model/pom.xml
+++ b/api-model/pom.xml
@@ -50,6 +50,11 @@
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>
     </dependency>
+    <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+    </dependency>
+
     <dependency>
       <groupId>ch.qos.logback</groupId>
       <artifactId>logback-classic</artifactId>
diff --git a/api-model/src/main/java/io/enmasse/address/model/Address.java b/api-model/src/main/java/io/enmasse/address/model/Address.java
index 34a7cd791..2ba546601 100644
--- a/api-model/src/main/java/io/enmasse/address/model/Address.java
+++ b/api-model/src/main/java/io/enmasse/address/model/Address.java
@@ -6,7 +6,6 @@ package io.enmasse.address.model;
 
 import java.nio.charset.StandardCharsets;
 import java.util.*;
-
 import javax.validation.Valid;
 import javax.validation.constraints.NotNull;
 
diff --git a/api-model/src/main/java/io/enmasse/address/model/AuthenticationService.java b/api-model/src/main/java/io/enmasse/address/model/AuthenticationService.java
index 3d5fedd93..18f5c7413 100644
--- a/api-model/src/main/java/io/enmasse/address/model/AuthenticationService.java
+++ b/api-model/src/main/java/io/enmasse/address/model/AuthenticationService.java
@@ -5,6 +5,8 @@
 package io.enmasse.address.model;
 
 import com.fasterxml.jackson.annotation.JsonInclude;
+import com.google.common.base.MoreObjects.ToStringHelper;
+
 import io.enmasse.admin.model.v1.AbstractWithAdditionalProperties;
 import io.fabric8.kubernetes.api.model.Doneable;
 import io.sundr.builder.annotations.Buildable;
@@ -78,4 +80,12 @@ public class AuthenticationService extends AbstractWithAdditionalProperties {
     public int hashCode() {
         return Objects.hash(name, type, overrides);
     }
+
+    @Override
+    protected ToStringHelper toStringHelper() {
+        return super.toStringHelper()
+                .add("name", this.name)
+                .add("type", this.type)
+                .add("overrides", this.overrides);
+    }
 }
diff --git a/api-model/src/main/java/io/enmasse/address/model/AuthenticationServiceSettings.java b/api-model/src/main/java/io/enmasse/address/model/AuthenticationServiceSettings.java
index 15ecacb1c..d951ea4ae 100644
--- a/api-model/src/main/java/io/enmasse/address/model/AuthenticationServiceSettings.java
+++ b/api-model/src/main/java/io/enmasse/address/model/AuthenticationServiceSettings.java
@@ -5,6 +5,8 @@
 package io.enmasse.address.model;
 
 import com.fasterxml.jackson.annotation.JsonInclude;
+import com.google.common.base.MoreObjects.ToStringHelper;
+
 import io.enmasse.admin.model.v1.AbstractWithAdditionalProperties;
 import io.fabric8.kubernetes.api.model.Doneable;
 import io.fabric8.kubernetes.api.model.SecretReference;
@@ -89,13 +91,13 @@ public class AuthenticationServiceSettings extends AbstractWithAdditionalPropert
     }
 
     @Override
-    public String toString() {
-        return "AuthenticationServiceOverrides{" +
-                "host='" + host + '\'' +
-                ", port=" + port +
-                ", realm='" + realm + '\'' +
-                ", caCertSecret='" + caCertSecret + '\'' +
-                ", clientCertSecret='" + realm + '\'' +
-                '}';
+    protected ToStringHelper toStringHelper() {
+        return super.toStringHelper()
+                .add("host", this.host)
+                .add("port", this.port)
+                .add("realm", this.realm)
+                .add("caCertSecert", this.caCertSecret)
+                .add("clientCertSecret", this.clientCertSecret);
     }
+
 }
diff --git a/api-model/src/main/java/io/enmasse/address/model/KubeUtil.java b/api-model/src/main/java/io/enmasse/address/model/KubeUtil.java
index 01e74e721..2753d9d7f 100644
--- a/api-model/src/main/java/io/enmasse/address/model/KubeUtil.java
+++ b/api-model/src/main/java/io/enmasse/address/model/KubeUtil.java
@@ -4,6 +4,7 @@
  */
 package io.enmasse.address.model;
 
+import java.nio.charset.StandardCharsets;
 import java.util.*;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
@@ -17,6 +18,9 @@ import io.fabric8.kubernetes.api.model.apps.StatefulSet;
  * Various static utilities that don't belong in a specific place
  */
 public final class KubeUtil {
+
+    private static final byte [] GO_NAMESPACE = new byte []{(byte)0x15,(byte)0x16,(byte)0xb2,(byte)0x46,(byte)0x23,(byte)0xaa,(byte)0x11,(byte)0xe9,(byte)0xb6,(byte)0x15,(byte)0xc8,(byte)0x5b,(byte)0x76,(byte)0x2e,(byte)0x5a,(byte)0x2c};
+
     private static final int MAX_KUBE_NAME = 63 - 3; // max length of identifier - space for pod identifier
     private static final Pattern addressPattern = Pattern.compile("[^a-z0-9\\-]");
     private static final Pattern usernamePattern = Pattern.compile("[^a-z0-9\\-.@_]");
@@ -32,7 +36,7 @@ public final class KubeUtil {
         return sanitizeWithPattern(name, usernamePattern);
     }
 
-    private static String sanitizeWithPattern(String value, Pattern pattern) {
+    public static String sanitizeWithPattern(String value, Pattern pattern) {
         if (value == null) {
             return null;
         }
@@ -56,8 +60,8 @@ public final class KubeUtil {
         return clean;
     }
 
-    public static String sanitizeWithUuid(String name, String uuid) {
-        name = sanitizeName(name);
+    public static String sanitizeWithUuid(String name, String uuid, Pattern pattern) {
+        name = sanitizeWithPattern(name, pattern);
         if (name.length() + uuid.length() + 1 > MAX_KUBE_NAME) {
             name = name.substring(0, MAX_KUBE_NAME - uuid.length() - 1);
         }
@@ -65,6 +69,33 @@ public final class KubeUtil {
         return name;
     }
 
+    public static String sanitizeWithUuid(String name, String uuid) {
+        return sanitizeWithUuid(name, uuid, addressPattern);
+    }
+
+    /**
+     * Create the address name, aligned with the Go logic of creating names.
+     * <br>
+     * The main difference between Go and the Java method {@link #generateName(String, String)} is, that in Go
+     * you must use a namespace prefix for a type 3 UUID. This method uses the same prefix as the Go code.
+     *
+     * @param addressSpace The name of the address space.
+     * @param address The name of the address.
+     * @return The encoded name, compatible with the logic in Go.
+     */
+    public static String sanitizeForGo(final String addressSpace, final String address) {
+
+        byte [] addressBytes = address.getBytes(StandardCharsets.UTF_8);
+        byte [] data = new byte[addressBytes.length + GO_NAMESPACE.length];
+
+        System.arraycopy(GO_NAMESPACE, 0, data, 0, GO_NAMESPACE.length);
+        System.arraycopy(addressBytes, 0, data, GO_NAMESPACE.length, addressBytes.length);
+
+        String uuid = UUID.nameUUIDFromBytes(data).toString();
+
+        return KubeUtil.sanitizeName(addressSpace) + "." + KubeUtil.sanitizeWithUuid(address, uuid, addressPattern);
+    }
+
     public static String getAddressSpaceCaSecretName(AddressSpace addressSpace) {
         return sanitizeName("ca-" + addressSpace.getMetadata().getName() + "." + addressSpace.getAnnotation(AnnotationKeys.INFRA_UUID));
     }
diff --git a/api-model/src/main/java/io/enmasse/admin/model/v1/AbstractWithAdditionalProperties.java b/api-model/src/main/java/io/enmasse/admin/model/v1/AbstractWithAdditionalProperties.java
index 7ddf8b52c..5be0d29f1 100644
--- a/api-model/src/main/java/io/enmasse/admin/model/v1/AbstractWithAdditionalProperties.java
+++ b/api-model/src/main/java/io/enmasse/admin/model/v1/AbstractWithAdditionalProperties.java
@@ -10,6 +10,8 @@ import java.util.Map;
 
 import com.fasterxml.jackson.annotation.JsonAnyGetter;
 import com.fasterxml.jackson.annotation.JsonAnySetter;
+import com.google.common.base.MoreObjects;
+import com.google.common.base.MoreObjects.ToStringHelper;
 
 import io.fabric8.kubernetes.api.model.Doneable;
 import io.sundr.builder.annotations.Buildable;
@@ -42,4 +44,13 @@ public abstract class AbstractWithAdditionalProperties implements WithAdditional
         this.additionalProperties.put(name, value);
     }
 
+    protected ToStringHelper toStringHelper() {
+        return MoreObjects.toStringHelper(this)
+                .add("additionalProperties", this.additionalProperties);
+    }
+
+    @Override
+    public String toString() {
+        return toStringHelper().toString();
+    }
 }
diff --git a/api-model/src/main/java/io/enmasse/iot/model/v1/OperatorConfig.java b/api-model/src/main/java/io/enmasse/iot/model/v1/OperatorConfig.java
new file mode 100644
index 000000000..bf4437233
--- /dev/null
+++ b/api-model/src/main/java/io/enmasse/iot/model/v1/OperatorConfig.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2019, EnMasse authors.
+ * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html).
+ */
+
+package io.enmasse.iot.model.v1;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+
+import io.fabric8.kubernetes.api.model.Doneable;
+import io.sundr.builder.annotations.Buildable;
+import io.sundr.builder.annotations.Inline;
+
+@Buildable(
+        editableEnabled = false,
+        generateBuilderPackage = false,
+        builderPackage = "io.fabric8.kubernetes.api.builder",
+        inline = @Inline(
+                type = Doneable.class,
+                prefix = "Doneable",
+                value = "done"))
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class OperatorConfig extends ServiceConfig {
+    private ContainerConfig container;
+
+    public ContainerConfig getContainer() {
+        return container;
+    }
+    public void setContainer(ContainerConfig container) {
+        this.container = container;
+    }
+}
diff --git a/api-model/src/main/java/io/enmasse/iot/model/v1/ServicesConfig.java b/api-model/src/main/java/io/enmasse/iot/model/v1/ServicesConfig.java
index fe5e4c890..4e31c99f7 100644
--- a/api-model/src/main/java/io/enmasse/iot/model/v1/ServicesConfig.java
+++ b/api-model/src/main/java/io/enmasse/iot/model/v1/ServicesConfig.java
@@ -27,6 +27,7 @@ public class ServicesConfig {
     private AuthenticationServiceConfig authentication;
     private TenantServiceConfig tenant;
     private CollectorConfig collector;
+    private OperatorConfig operator;
 
     public DeviceRegistryServiceConfig getDeviceRegistry() {
         return deviceRegistry;
@@ -56,5 +57,11 @@ public class ServicesConfig {
         this.collector = collector;
     }
 
+    public OperatorConfig getOperator() {
+        return operator;
+    }
+    public void setOperator(OperatorConfig operator) {
+        this.operator = operator;
+    }
 
 }
diff --git a/api-server/src/main/java/io/enmasse/api/v1/http/HttpUserService.java b/api-server/src/main/java/io/enmasse/api/v1/http/HttpUserService.java
index 191016867..e8a858e3f 100644
--- a/api-server/src/main/java/io/enmasse/api/v1/http/HttpUserService.java
+++ b/api-server/src/main/java/io/enmasse/api/v1/http/HttpUserService.java
@@ -49,7 +49,6 @@ import io.enmasse.user.model.v1.User;
 import io.enmasse.user.model.v1.UserBuilder;
 import io.enmasse.user.model.v1.UserCrd;
 import io.enmasse.user.model.v1.UserList;
-import io.enmasse.user.model.v1.UserSpecBuilder;
 import io.fabric8.kubernetes.api.model.ListMeta;
 import io.fabric8.kubernetes.api.model.ObjectMetaBuilder;
 
@@ -195,9 +194,9 @@ public class HttpUserService {
     private User setUserDefaults(User user, String namespace) {
         if (user.getMetadata().getNamespace() == null) {
             user = new UserBuilder(user)
-                    .withMetadata(new ObjectMetaBuilder(user.getMetadata())
-                            .withNamespace(namespace)
-                            .build())
+                    .editOrNewMetadata()
+                    .withNamespace(namespace)
+                    .endMetadata()
                     .build();
         }
         return user;
@@ -301,13 +300,20 @@ public class HttpUserService {
     }
 
     private User overrideNameAndNamespace(User existingUser, User replacement) {
-        replacement = new UserBuilder(replacement)
-                .withSpec(new UserSpecBuilder(replacement.getSpec())
-                        .withUsername(existingUser.getSpec().getUsername())
-                        .build())
-                .withMetadata(new ObjectMetaBuilder(existingUser.getMetadata())
-                        .withNamespace(existingUser.getMetadata().getNamespace()).build()).build();
-        return replacement;
+
+        return new UserBuilder(replacement)
+
+                .editOrNewSpec()
+                .withUsername(existingUser.getSpec().getUsername())
+                .endSpec()
+
+                .editOrNewMetadata()
+                .withName(existingUser.getMetadata().getName())
+                .withNamespace(existingUser.getMetadata().getNamespace())
+                .endMetadata()
+
+                .build();
+
     }
 
     @DELETE
diff --git a/broker-plugin/plugin/src/main/resources/standard/sharded-queue/README.md b/broker-plugin/plugin/src/main/resources/standard/sharded-queue/README.md
new file mode 100644
index 000000000..b11055caa
--- /dev/null
+++ b/broker-plugin/plugin/src/main/resources/standard/sharded-queue/README.md
@@ -0,0 +1,3 @@
+*NOTE*: The `broker.xml` file in this folder is needed to support upgrading
+from 0.26.X where sharded queues were deployed using this broker.xml rather 
+than as a colocated broker which is the case after 0.26.x.
diff --git a/broker-plugin/plugin/src/main/resources/standard/sharded-queue/broker.xml b/broker-plugin/plugin/src/main/resources/standard/sharded-queue/broker.xml
new file mode 100644
index 000000000..7bddd3c96
--- /dev/null
+++ b/broker-plugin/plugin/src/main/resources/standard/sharded-queue/broker.xml
@@ -0,0 +1,150 @@
+<?xml version='1.0'?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+<!--
+NOTE: This file is needed to support upgrading from 0.26.X where sharded 
+queues were deployed using this broker.xml rather than as a colocated 
+broker which is the case after 0.26.x.
+-->
+
+<configuration xmlns="urn:activemq"
+               xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+               xsi:schemaLocation="urn:activemq /schema/artemis-configuration.xsd">
+
+   <core xmlns="urn:activemq:core" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="urn:activemq:core ">
+
+      <name>$CONTAINER_ID</name>
+
+      <persistence-enabled>true</persistence-enabled>
+
+      <!-- this could be ASYNCIO or NIO
+       -->
+      <journal-type>ASYNCIO</journal-type>
+
+      <paging-directory>./data/paging</paging-directory>
+
+      <bindings-directory>./data/bindings</bindings-directory>
+
+      <journal-directory>./data/journal</journal-directory>
+
+      <large-messages-directory>./data/large-messages</large-messages-directory>
+
+      <journal-datasync>true</journal-datasync>
+
+      <journal-min-files>2</journal-min-files>
+
+      <journal-pool-files>-1</journal-pool-files>
+
+      <journal-file-size>10M</journal-file-size>
+
+      <journal-buffer-timeout>2212000</journal-buffer-timeout>
+
+
+      <!-- how often we are looking for how many bytes are being used on the disk in ms -->
+      <disk-scan-period>5000</disk-scan-period>
+
+      <!-- once the disk hits this limit the system will block, or close the connection in certain protocols
+           that won't support flow control. -->
+      <max-disk-usage>90</max-disk-usage>
+
+      <!-- should the broker detect dead locks and other issues -->
+      <critical-analyzer>true</critical-analyzer>
+
+      <critical-analyzer-timeout>120000</critical-analyzer-timeout>
+
+      <critical-analyzer-check-period>60000</critical-analyzer-check-period>
+
+      <critical-analyzer-policy>HALT</critical-analyzer-policy>
+
+      <global-max-size>${GLOBAL_MAX_SIZE}</global-max-size>
+
+      <acceptors>
+         <acceptor name="artemis">tcp://0.0.0.0:61616?tcpSendBufferSize=1048576;tcpReceiveBufferSize=1048576;protocols=CORE</acceptor>
+         <acceptor name="amqp">tcp://0.0.0.0:5673?protocols=AMQP;sslEnabled=true;keyStorePath=${KEYSTORE_PATH};keyStorePassword=enmasse;trustStorePath=${TRUSTSTORE_PATH};trustStorePassword=enmasse;verifyHost=false;needClientAuth=true</acceptor>
+      </acceptors>
+
+
+      <security-settings>
+         <security-setting-plugin class-name="io.enmasse.artemis.sasl_delegation.SaslGroupBasedSecuritySettingsPlugin">
+            <setting name="name" value="enmasse"/>
+            <setting name="useGroupsFromSaslDelegation" value="$KEYCLOAK_GROUP_PERMISSIONS"/>
+         </security-setting-plugin>
+      </security-settings>
+
+      <address-settings>
+         <!--default for catch all-->
+         <address-setting match="#">
+            <max-delivery-attempts>-1</max-delivery-attempts>
+            <redelivery-delay>1</redelivery-delay>
+            <redelivery-delay-multiplier>1.5</redelivery-delay-multiplier>
+            <max-redelivery-delay>10000</max-redelivery-delay>
+            <dead-letter-address>!!GLOBAL_DLQ</dead-letter-address>
+            <!-- with -1 only the global-max-size is in use for limiting -->
+            <max-size-bytes>-1</max-size-bytes>
+            <message-counter-history-day-limit>10</message-counter-history-day-limit>
+            <address-full-policy>${ADDRESS_FULL_POLICY}</address-full-policy>
+         </address-setting>
+      </address-settings>
+      <addresses>
+         <address name="!!GLOBAL_DLQ">
+           <anycast>
+             <queue name="!!GLOBAL_DLQ" />
+           </anycast>
+         </address>
+      </addresses>
+      <wildcard-addresses>
+        <enabled>true</enabled>
+        <delimiter>/</delimiter>
+        <any-words>#</any-words>
+        <single-word>+</single-word>
+      </wildcard-addresses>
+      <connector-services>
+        <connector-service name="$${dummy}override-amqp-connector-in">
+          <factory-class>org.apache.activemq.artemis.integration.amqp.AMQPConnectorServiceFactory</factory-class>
+          <param key="host" value="$MESSAGING_SERVICE_HOST" />
+          <param key="port" value="$MESSAGING_SERVICE_PORT_AMQPS_BROKER" />
+          <param key="containerId" value="$QUEUE_NAME-in" />
+          <param key="clusterId" value="$QUEUE_NAME" />
+          <param key="idleTimeoutMs" value="$CONNECTOR_IN_IDLE_TIMEOUT_MS" />
+          <param key="nettyThreads" value="$CONNECTOR_IN_NETTY_THREADS" />
+        </connector-service>
+        <connector-service name="$${dummy}override-amqp-connector-out">
+          <factory-class>org.apache.activemq.artemis.integration.amqp.AMQPConnectorServiceFactory</factory-class>
+          <param key="host" value="$MESSAGING_SERVICE_HOST" />
+          <param key="port" value="$MESSAGING_SERVICE_PORT_AMQPS_BROKER" />
+          <param key="containerId" value="$QUEUE_NAME-out" />
+          <param key="clusterId" value="$QUEUE_NAME" />
+          <param key="idleTimeoutMs" value="$CONNECTOR_OUT_IDLE_TIMEOUT_MS" />
+          <param key="nettyThreads" value="$CONNECTOR_OUT_NETTY_THREADS" />
+        </connector-service>
+        <connector-service name="$${dummy}override-dlq-out">
+          <factory-class>org.apache.activemq.artemis.integration.amqp.AMQPConnectorServiceFactory</factory-class>
+          <param key="host" value="$MESSAGING_SERVICE_HOST" />
+          <param key="port" value="$MESSAGING_SERVICE_PORT_AMQPS_BROKER" />
+          <param key="containerId" value="broker-global-dlq-out" />
+          <param key="clusterId" value="$QUEUE_NAME" />
+          <param key="idleTimeoutMs" value="$CONNECTOR_OUT_IDLE_TIMEOUT_MS" />
+          <param key="nettyThreads" value="$CONNECTOR_OUT_NETTY_THREADS" />
+        </connector-service>
+      </connector-services>
+   </core>
+</configuration>
+
diff --git a/cmd/iot-proxy-configurator/configurator.go b/cmd/iot-proxy-configurator/configurator.go
index 3fc0b822b..0b718019f 100644
--- a/cmd/iot-proxy-configurator/configurator.go
+++ b/cmd/iot-proxy-configurator/configurator.go
@@ -205,19 +205,33 @@ func (c *Configurator) syncHandler(key string) error {
 
 			log.Info("Item got deleted. Deleting configuration.")
 
-			err = c.deleteProject(&metav1.ObjectMeta{
+			// if something went wrong deleting, then returning
+			// and error will re-queue the item
+
+			return c.deleteProject(&metav1.ObjectMeta{
 				Namespace: namespace,
 				Name:      name,
 			})
 
-			// if something went wrong deleting, then returning
-			// and error will re-queue the item
-			return err
 		}
 
 		return err
 	}
 
+	// check if we are scheduled for deletion ...
+	if project.DeletionTimestamp != nil {
+
+		// ... yes we are, go ahead and delete
+
+		log.Info("Item scheduled for deletion. Deleting configuration.")
+
+		return c.deleteProject(&metav1.ObjectMeta{
+			Namespace: namespace,
+			Name:      name,
+		})
+
+	}
+
 	if !project.Status.IsReady || project.Status.DownstreamEndpoint == nil {
 		// project is not ready yet
 		return nil
diff --git a/documentation/modules/con-console.adoc b/documentation/modules/con-console.adoc
index c77314fe3..9c70f8efa 100644
--- a/documentation/modules/con-console.adoc
+++ b/documentation/modules/con-console.adoc
@@ -5,7 +5,7 @@
 [id='con-console-{context}']
 = {ConsoleName} user permissions
 
-{ConsoleName} uses the OpenShift RBAC permissions model. For more information about the OpenShift RBAC permissions model, see the link:https://docs.openshift.com/container-platform/3.11/admin_guide/manage_rbac.html[OpenShift documentation^].
+{ConsoleName} uses the OpenShift RBAC permissions model. For more information about the OpenShift RBAC permissions model, see the link:https://access.redhat.com/documentation/en-us/openshift_container_platform/3.11/html-single/cluster_administration/index#admin-guide-manage-rbac[OpenShift 3.11 documentation^].
 
 To use {ConsoleName}, the OpenShift user requires a role that grants access to `addressspace` and `address` resources. For example, for edit access, `edit` permissions need be to given to the associated role object, and for view-only access, `list` permissions need to be granted. 
 
@@ -14,6 +14,6 @@ For more information about the {ProductName} example roles, see link:{BookUrlBas
 endif::SingleBookLink[]
 
 ifndef::SingleBookLink[]
-For more information about the {ProductName} example roles, see link:{BookUrlBase}{BaseProductVersion}/html-single/installing_and_managing_amq_online_on_openshift_container_platform/#ref-example-roles-messaging[{ProductName} example roles].
+For more information about the {ProductName} example roles, see link:{BookUrlBase}{BaseProductVersion}/html-single/installing_and_managing_amq_online_on_openshift/#ref-example-roles-messaging[{ProductName} example roles].
 endif::SingleBookLink[]
 
diff --git a/documentation/modules/ref-address-space-example-external-auth-service-override.adoc b/documentation/modules/ref-address-space-example-external-auth-service-override.adoc
index d155bf258..6df2d3a8a 100644
--- a/documentation/modules/ref-address-space-example-external-auth-service-override.adoc
+++ b/documentation/modules/ref-address-space-example-external-auth-service-override.adoc
@@ -12,7 +12,7 @@ For more information about how to configure an external authentication service t
 endif::SingleBookLink[]
 
 ifndef::SingleBookLink[]
-For more information about how to configure an external authentication service to allow a messaging tenant to override host name, port number, and realm, see link:{BookUrlBase}{BaseProductVersion}/html-single/installing_and_managing_amq_online_on_openshift_container_platform/#ref-external-auth-service-example-allow-overrides-messaging[External authentication service example allowing overrides].
+For more information about how to configure an external authentication service to allow a messaging tenant to override host name, port number, and realm, see link:{BookUrlBase}{BaseProductVersion}/html-single/installing_and_managing_amq_online_on_openshift/#ref-external-auth-service-example-allow-overrides-messaging[External authentication service example allowing overrides].
 endif::SingleBookLink[]
 
 [source,yaml,options="nowrap",subs="attributes+"]
diff --git a/documentation/modules/ref-example-roles.adoc b/documentation/modules/ref-example-roles.adoc
index 1d374387f..cfadd63de 100644
--- a/documentation/modules/ref-example-roles.adoc
+++ b/documentation/modules/ref-example-roles.adoc
@@ -14,7 +14,7 @@ For more information about messaging tenant resources, see the link:{BookUrlBase
 endif::SingleBookLink[]
 
 ifndef::SingleBookLink[]
-For more information about messaging tenant resources, see the link:{BookUrlBase}{BaseProductVersion}/html-single/using_amq_online_on_openshift_container_platform/#ref-resources-table-messaging-tenant-messaging[{ProductName} messaging tenant resources table].
+For more information about messaging tenant resources, see the link:{BookUrlBase}{BaseProductVersion}/html-single/using_amq_online_on_openshift/#ref-resources-table-messaging-tenant-messaging[{ProductName} messaging tenant resources table].
 endif::SingleBookLink[]
 
 .{ProductName} example roles table
diff --git a/keycloak-user-api/src/main/java/io/enmasse/user/keycloak/KeycloakUserApi.java b/keycloak-user-api/src/main/java/io/enmasse/user/keycloak/KeycloakUserApi.java
index 498123628..3e58c4039 100644
--- a/keycloak-user-api/src/main/java/io/enmasse/user/keycloak/KeycloakUserApi.java
+++ b/keycloak-user-api/src/main/java/io/enmasse/user/keycloak/KeycloakUserApi.java
@@ -5,6 +5,7 @@
 package io.enmasse.user.keycloak;
 
 
+import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import io.enmasse.admin.model.v1.AuthenticationService;
 import io.enmasse.k8s.util.TimeUtil;
@@ -40,6 +41,9 @@ import static java.util.Optional.empty;
 
 public class KeycloakUserApi implements UserApi, RealmApi {
 
+    private static final String ATTR_ANNOTATIONS = "annotations";
+    private static final String ATTR_OWNER_REFERENCES = "ownerReferences";
+
     private static final Logger log = LoggerFactory.getLogger(KeycloakUserApi.class);
 
     private final Clock clock;
@@ -160,16 +164,37 @@ public class KeycloakUserApi implements UserApi, RealmApi {
         attributes.put("authenticationType", Collections.singletonList(user.getSpec().getAuthentication().getType().name()));
 
         final List<String> ownerReferences = ownerReferencesToString(user.getMetadata().getOwnerReferences());
-        attributes.put("ownerReferences", ownerReferences);
+        attributes.put(ATTR_OWNER_REFERENCES, ownerReferences);
 
         Instant now = clock.instant();
         attributes.put("creationTimestamp", Collections.singletonList(TimeUtil.formatRfc3339(now)));
+        attributes.put(ATTR_ANNOTATIONS, annotationsToString(user.getMetadata().getAnnotations()));
 
         userRep.setAttributes(attributes);
 
         return userRep;
     }
 
+    static List<String> annotationsToString(final Map<String, String> annotations) {
+
+        if (annotations == null) {
+            return null;
+        }
+
+        final ObjectMapper mapper = new ObjectMapper();
+
+        return annotations.entrySet().stream()
+                .map(entry -> {
+                    try {
+                        return mapper.writeValueAsString(entry);
+                    } catch (Exception e) {
+                        throw new RuntimeException(e);
+                    }
+                })
+                .collect(Collectors.toList());
+
+    }
+
     private static List<String> ownerReferencesToString(final List<OwnerReference> references) {
 
         if (references == null) {
@@ -178,7 +203,7 @@ public class KeycloakUserApi implements UserApi, RealmApi {
 
         final ObjectMapper mapper = new ObjectMapper();
 
-        final List<String> ownerReferences = references
+        return references
                 .stream()
                 .map(ownerReference -> {
                     try {
@@ -189,12 +214,31 @@ public class KeycloakUserApi implements UserApi, RealmApi {
                 })
                 .collect(Collectors.toList());
 
-        return ownerReferences;
+    }
+
+    static Map<String, String> annotationsFromString(final List<String> annotationValues) {
+
+        if (annotationValues == null || annotationValues.isEmpty()) {
+            return null;
+        }
+
+        final ObjectMapper mapper = new ObjectMapper();
+
+        return annotationValues
+                .stream().<Map.Entry<String, String>>map(ref -> {
+                    try {
+                        return mapper.readValue(ref, new TypeReference<Map.Entry<String, String>>() {});
+                    } catch (IOException e) {
+                        throw new RuntimeException(e);
+                    }
+                })
+                .collect(Collectors.<Map.Entry<String, String>, String, String>toMap(Map.Entry::getKey, Map.Entry::getValue));
+
     }
 
     private static List<OwnerReference> ownerReferencesFromString(final List<String> attributeValues) {
 
-        if (attributeValues == null) {
+        if (attributeValues == null || attributeValues.isEmpty() ) {
             return null;
         }
 
@@ -390,10 +434,47 @@ public class KeycloakUserApi implements UserApi, RealmApi {
                 }
             }
             applyAuthorizationRules(realm, user, realm.users().get(userRep.getId()));
+            applyAttributes(realm, user, userRep);
             return true;
         });
     }
 
+    private void applyAttributes(RealmResource realm, User user, UserRepresentation userRep) {
+        boolean changed = false;
+        if (updateAttribute(userRep, ATTR_ANNOTATIONS, annotationsToString(user.getMetadata().getAnnotations()))) {
+            changed = true;
+        }
+        if (updateAttribute(userRep, ATTR_OWNER_REFERENCES, ownerReferencesToString(user.getMetadata().getOwnerReferences()))) {
+            changed = true;
+        }
+        if (changed) {
+            realm.users().get(userRep.getId()).update(userRep);
+        }
+    }
+
+    private static boolean updateAttribute(final UserRepresentation userRep, final String key, List<String> value) {
+
+        final String[] currentValue = Optional.ofNullable(userRep.getAttributes().get(key))
+                .map(v -> v.toArray(String[]::new))
+                .orElseGet(()-> new String[] {});
+
+        final String[] newValue = Optional.ofNullable(value)
+                .map(v-> v.toArray(String[]::new))
+                .orElseGet(()-> new String[] {});
+
+        if (Arrays.equals(currentValue, newValue)) {
+            return false;
+        }
+
+        if (value.isEmpty()) {
+            userRep.getAttributes().remove(key);
+        } else {
+            userRep.getAttributes().put(key, value);
+        }
+
+        return true;
+    }
+
     private Optional<String> getGroupId(List<GroupRepresentation> groupRepresentations, String groupName) {
         for (GroupRepresentation groupRepresentation : groupRepresentations) {
             if (groupName.equals(groupRepresentation.getName())) {
@@ -622,7 +703,8 @@ public class KeycloakUserApi implements UserApi, RealmApi {
                         .withNamespace(namespace)
                         .withSelfLink("/apis/user.enmasse.io/v1beta1/namespaces/" + namespace + "/messagingusers/" + name)
                         .withCreationTimestamp(userRep.getAttributes().get("creationTimestamp").get(0))
-                        .withOwnerReferences(ownerReferencesFromString(userRep.getAttributes().get("ownerReferences")))
+                        .withOwnerReferences(ownerReferencesFromString(userRep.getAttributes().get(ATTR_OWNER_REFERENCES)))
+                        .withAnnotations(annotationsFromString(userRep.getAttributes().get(ATTR_ANNOTATIONS)))
                         .build())
                 .withSpec(new UserSpecBuilder()
                         .withUsername(userRep.getUsername())
diff --git a/pkg/apis/iot/v1alpha1/types_project.go b/pkg/apis/iot/v1alpha1/types_project.go
index 2c64028f0..2ee374893 100644
--- a/pkg/apis/iot/v1alpha1/types_project.go
+++ b/pkg/apis/iot/v1alpha1/types_project.go
@@ -29,9 +29,14 @@ type IoTProjectSpec struct {
 }
 
 type IoTProjectStatus struct {
-	IsReady            bool                        `json:"isReady"`
-	TenantName         string                      `json:"tenantName"`
-	DownstreamEndpoint *ExternalDownstreamStrategy `json:"downstreamEndpoint,omitempty"`
+	IsReady     bool   `json:"isReady"`
+	Phase       string `json:"phase"`
+	PhaseReason string `json:"phaseReason,omitempty"`
+
+	TenantName         string                 `json:"tenantName"`
+	DownstreamEndpoint *ConnectionInformation `json:"downstreamEndpoint,omitempty"`
+
+	Managed *ManagedStatus `json:"managed,omitempty"`
 
 	Conditions []ProjectCondition `json:"conditions"`
 }
@@ -89,7 +94,7 @@ type AddressConfig struct {
 	Type string `json:"type,omitempty"`
 }
 
-type ExternalDownstreamStrategy struct {
+type ConnectionInformation struct {
 	Host string `json:"host"`
 	Port uint16 `json:"port"`
 
@@ -99,6 +104,15 @@ type ExternalDownstreamStrategy struct {
 	Certificate []byte `json:"certificate,omitempty"`
 }
 
+type ExternalDownstreamStrategy struct {
+	ConnectionInformation `json:",inline"`
+}
+
+type ManagedStatus struct {
+	PasswordTime metav1.Time `json:"passwordTime,omitempty"`
+	AddressSpace string      `json:"addressSpace,omitempty"`
+}
+
 type Credentials struct {
 	Username string `json:"username"`
 	Password string `json:"password"`
diff --git a/pkg/apis/iot/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/iot/v1alpha1/zz_generated.deepcopy.go
index b605ebe72..c1d087dbd 100644
--- a/pkg/apis/iot/v1alpha1/zz_generated.deepcopy.go
+++ b/pkg/apis/iot/v1alpha1/zz_generated.deepcopy.go
@@ -316,6 +316,28 @@ func (in *CommonStatus) DeepCopy() *CommonStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ConnectionInformation) DeepCopyInto(out *ConnectionInformation) {
+	*out = *in
+	out.Credentials = in.Credentials
+	if in.Certificate != nil {
+		in, out := &in.Certificate, &out.Certificate
+		*out = make([]byte, len(*in))
+		copy(*out, *in)
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConnectionInformation.
+func (in *ConnectionInformation) DeepCopy() *ConnectionInformation {
+	if in == nil {
+		return nil
+	}
+	out := new(ConnectionInformation)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ContainerConfig) DeepCopyInto(out *ContainerConfig) {
 	*out = *in
@@ -446,12 +468,7 @@ func (in *ExternalCacheNames) DeepCopy() *ExternalCacheNames {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ExternalDownstreamStrategy) DeepCopyInto(out *ExternalDownstreamStrategy) {
 	*out = *in
-	out.Credentials = in.Credentials
-	if in.Certificate != nil {
-		in, out := &in.Certificate, &out.Certificate
-		*out = make([]byte, len(*in))
-		copy(*out, *in)
-	}
+	in.ConnectionInformation.DeepCopyInto(&out.ConnectionInformation)
 	return
 }
 
@@ -811,7 +828,12 @@ func (in *IoTProjectStatus) DeepCopyInto(out *IoTProjectStatus) {
 	*out = *in
 	if in.DownstreamEndpoint != nil {
 		in, out := &in.DownstreamEndpoint, &out.DownstreamEndpoint
-		*out = new(ExternalDownstreamStrategy)
+		*out = new(ConnectionInformation)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.Managed != nil {
+		in, out := &in.Managed, &out.Managed
+		*out = new(ManagedStatus)
 		(*in).DeepCopyInto(*out)
 	}
 	if in.Conditions != nil {
@@ -937,6 +959,23 @@ func (in *ManagedDownstreamStrategy) DeepCopy() *ManagedDownstreamStrategy {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ManagedStatus) DeepCopyInto(out *ManagedStatus) {
+	*out = *in
+	in.PasswordTime.DeepCopyInto(&out.PasswordTime)
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ManagedStatus.
+func (in *ManagedStatus) DeepCopy() *ManagedStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(ManagedStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *MqttAdapterConfig) DeepCopyInto(out *MqttAdapterConfig) {
 	*out = *in
diff --git a/pkg/controller/iotproject/external.go b/pkg/controller/iotproject/external.go
index 7260e5108..426185eeb 100644
--- a/pkg/controller/iotproject/external.go
+++ b/pkg/controller/iotproject/external.go
@@ -8,12 +8,36 @@ package iotproject
 import (
 	"context"
 
+	"github.com/enmasseproject/enmasse/pkg/util/recon"
+
 	iotv1alpha1 "github.com/enmasseproject/enmasse/pkg/apis/iot/v1alpha1"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 )
 
 func (r *ReconcileIoTProject) reconcileExternal(ctx context.Context, request *reconcile.Request, project *iotv1alpha1.IoTProject) (reconcile.Result, error) {
 
+	if project.Status.Managed != nil {
+
+		project.Status.Phase = "Reconfiguring"
+		project.Status.PhaseReason = "Change of downstream strategy"
+
+		rc := recon.ReconcileContext{}
+
+		rc.Process(func() (result reconcile.Result, e error) {
+			return cleanupManagedResources(ctx, r.client, project)
+		})
+		if rc.Error() == nil {
+			project.Status.Managed = nil
+			rc.ProcessSimple(func() error {
+				return r.client.Status().Update(ctx, project)
+			})
+		}
+
+		// always re-queue when the cleared out managed resources
+
+		return reconcile.Result{Requeue: true}, nil
+	}
+
 	// we simply copy over the external information
 	// so everything is expected to be ready
 
@@ -27,7 +51,7 @@ func (r *ReconcileIoTProject) reconcileExternal(ctx context.Context, request *re
 		GetProjectCondition(iotv1alpha1.ProjectConditionTypeReady).
 		SetStatusOk()
 
-	project.Status.DownstreamEndpoint = project.Spec.DownstreamStrategy.ExternalDownstreamStrategy.DeepCopy()
+	project.Status.DownstreamEndpoint = project.Spec.DownstreamStrategy.ExternalDownstreamStrategy.ConnectionInformation.DeepCopy()
 	project.Status.IsReady = true
 
 	return reconcile.Result{}, nil
diff --git a/pkg/controller/iotproject/finalizers.go b/pkg/controller/iotproject/finalizers.go
index 854be18a2..7d2c9ede9 100644
--- a/pkg/controller/iotproject/finalizers.go
+++ b/pkg/controller/iotproject/finalizers.go
@@ -8,7 +8,10 @@ package iotproject
 import (
 	"context"
 	"fmt"
-	"time"
+
+	"github.com/enmasseproject/enmasse/pkg/util"
+
+	"k8s.io/apimachinery/pkg/api/errors"
 
 	"github.com/enmasseproject/enmasse/pkg/util/install"
 
@@ -53,37 +56,86 @@ func deconstructManagedResources(project *iotv1alpha1.IoTProject, ctx finalizer.
 
 	rc := recon.ReconcileContext{}
 
+	if project.Status.Managed != nil {
+		rc.Process(func() (result reconcile.Result, e error) {
+			return cleanupManagedResources(ctx.Context, ctx.Client, project)
+		})
+	}
+
+	return rc.Result()
+
+}
+
+// delete all managed resources by the project
+func cleanupManagedResources(ctx context.Context, c client.Client, project *iotv1alpha1.IoTProject) (reconcile.Result, error) {
+
+	rc := recon.ReconcileContext{}
+
+	managed := project.Status.Managed
+
+	if managed == nil {
+		return rc.Result()
+	}
+
+	for _, a := range Addresses {
+		rc.Process(func() (result reconcile.Result, e error) {
+
+			addressName := util.AddressName(project, a)
+			addressMetaName := util.EncodeAddressSpaceAsMetaName(managed.AddressSpace, addressName)
+
+			return cleanupResource(ctx, c, project, client.ObjectKey{
+				Namespace: project.Namespace,
+				Name:      addressMetaName,
+			}, &v1beta1.Address{})
+
+		})
+	}
+
 	rc.Process(func() (result reconcile.Result, e error) {
-		return collectResources(ctx.Context, ctx.Client, project, &v1beta1.AddressList{})
-	})
-	rc.Process(func() (result reconcile.Result, e error) {
-		return collectResources(ctx.Context, ctx.Client, project, &v1beta1.AddressSpaceList{})
+		return cleanupResource(ctx, c, project, client.ObjectKey{
+			Namespace: project.Namespace,
+			Name:      managed.AddressSpace,
+		}, &v1beta1.AddressSpace{})
 	})
+
 	rc.Process(func() (result reconcile.Result, e error) {
-		return collectResources(ctx.Context, ctx.Client, project, &userv1beta1.MessagingUserList{})
+		return cleanupResource(ctx, c, project, client.ObjectKey{
+			Namespace: project.Namespace,
+			Name:      managed.AddressSpace + "." + string(project.UID),
+		}, &userv1beta1.MessagingUser{})
 	})
 
 	return rc.Result()
 
 }
 
-// collect all resource which are owned by this project
-func collectResources(ctx context.Context, c client.Client, project *iotv1alpha1.IoTProject, list runtime.Object) (reconcile.Result, error) {
+func cleanupResource(ctx context.Context, c client.Client, project *iotv1alpha1.IoTProject, key client.ObjectKey, obj runtime.Object) (reconcile.Result, error) {
 
-	n, err := install.BulkRemoveOwner(ctx, c, project, false, list, client.ListOptions{
-		Namespace: project.Namespace,
-	})
+	err := c.Get(ctx, key, obj)
+
+	if err != nil {
+
+		if errors.IsNotFound(err) {
+			return reconcile.Result{}, nil
+		} else {
+			return reconcile.Result{}, err
+		}
+
+	}
+
+	result, err := install.RemoveAsOwner(project, obj, false)
 
 	if err != nil {
 		return reconcile.Result{}, err
 	}
 
-	if n > 0 {
-		return reconcile.Result{
-			RequeueAfter: 10 * time.Second,
-		}, nil
-	} else {
-		return reconcile.Result{}, nil
+	switch result {
+	case install.Found:
+		err = c.Update(ctx, obj)
+	case install.FoundAndEmpty:
+		err = c.Delete(ctx, obj)
 	}
 
+	return reconcile.Result{}, err
+
 }
diff --git a/pkg/controller/iotproject/iotproject_controller.go b/pkg/controller/iotproject/iotproject_controller.go
index 31dd70507..958b7a1ef 100644
--- a/pkg/controller/iotproject/iotproject_controller.go
+++ b/pkg/controller/iotproject/iotproject_controller.go
@@ -8,7 +8,6 @@ package iotproject
 import (
 	"context"
 	"fmt"
-	"time"
 
 	corev1 "k8s.io/api/core/v1"
 
@@ -16,8 +15,6 @@ import (
 
 	"github.com/enmasseproject/enmasse/pkg/util/recon"
 
-	"github.com/enmasseproject/enmasse/pkg/util"
-
 	enmassev1beta1 "github.com/enmasseproject/enmasse/pkg/apis/enmasse/v1beta1"
 	iotv1alpha1 "github.com/enmasseproject/enmasse/pkg/apis/iot/v1alpha1"
 	userv1beta1 "github.com/enmasseproject/enmasse/pkg/apis/user/v1beta1"
@@ -145,37 +142,6 @@ type ReconcileIoTProject struct {
 	scheme *runtime.Scheme
 }
 
-func (r *ReconcileIoTProject) updateProjectStatusError(ctx context.Context, request *reconcile.Request, project *iotv1alpha1.IoTProject, err error) error {
-
-	newProject := project.DeepCopy()
-	newProject.Status.IsReady = false
-
-	readyCondition := newProject.Status.GetProjectCondition(iotv1alpha1.ProjectConditionTypeReady)
-	var reason = ""
-	var message = ""
-	if err != nil {
-		reason = "ProcessingError"
-		message = err.Error()
-	}
-	readyCondition.SetStatus(corev1.ConditionFalse, reason, message)
-
-	return r.client.Update(ctx, newProject)
-}
-
-func (r *ReconcileIoTProject) updateProjectStatusReady(ctx context.Context, request *reconcile.Request, project *iotv1alpha1.IoTProject, endpointStatus *iotv1alpha1.ExternalDownstreamStrategy) error {
-
-	newProject := project.DeepCopy()
-
-	newProject.Status.IsReady = true
-	newProject.Status.DownstreamEndpoint = endpointStatus.DeepCopy()
-
-	newProject.Status.
-		GetProjectCondition(iotv1alpha1.ProjectConditionTypeReady).
-		SetStatusOk()
-
-	return r.client.Update(ctx, newProject)
-}
-
 func (r *ReconcileIoTProject) updateProjectStatus(ctx context.Context, project *iotv1alpha1.IoTProject, currentError error) (reconcile.Result, error) {
 
 	newProject := project.DeepCopy()
@@ -204,6 +170,8 @@ func (r *ReconcileIoTProject) updateProjectStatus(ctx context.Context, project *
 	}
 	if newProject.Status.IsReady {
 		status = corev1.ConditionTrue
+		newProject.Status.Phase = "Ready"
+		newProject.Status.PhaseReason = ""
 	} else {
 		status = corev1.ConditionFalse
 	}
@@ -219,25 +187,6 @@ func (r *ReconcileIoTProject) updateProjectStatus(ctx context.Context, project *
 
 }
 
-func (r *ReconcileIoTProject) applyUpdate(ctx context.Context, status *iotv1alpha1.ExternalDownstreamStrategy, err error, request *reconcile.Request, project *iotv1alpha1.IoTProject) (reconcile.Result, error) {
-
-	if err != nil {
-
-		if util.IsNotReadyYetError(err) {
-			log.Info("Not ready yet, re-scheduling...")
-			// if the resource is not ready yet, then we retry after a delay
-			return reconcile.Result{Requeue: true, RequeueAfter: time.Minute}, nil
-		}
-
-		log.Error(err, "failed to reconcile")
-		_ = r.updateProjectStatusError(ctx, request, project, err)
-		return reconcile.Result{}, err
-	}
-
-	err = r.updateProjectStatusReady(ctx, request, project, status)
-	return reconcile.Result{}, err
-}
-
 // Reconcile by reading the IoT project spec and making required changes
 //
 // returning an error will get the request re-queued
diff --git a/pkg/controller/iotproject/managed.go b/pkg/controller/iotproject/managed.go
index ab5cd2e25..a085e5b8c 100644
--- a/pkg/controller/iotproject/managed.go
+++ b/pkg/controller/iotproject/managed.go
@@ -30,14 +30,25 @@ const AddressNameCommandLegacy = "control"
 const AddressNameCommand = "command"
 const AddressNameCommandResponse = "command_response"
 
+var Addresses = []string{
+	AddressNameTelemetry,
+	AddressNameEvent,
+	AddressNameCommandLegacy,
+	AddressNameCommand,
+	AddressNameCommandResponse,
+}
+
 const resourceTypeAddressSpace = "Address Space"
 const resourceTypeAdapterUser = "Adapter User"
 
+const annotationPasswordSyncTime = annotationBase + "/passwordSyncTime"
+const annotationProject = annotationBase + "/project.name"
+const annotationProjectUID = annotationBase + "/project.uid"
+
 type managedStatus struct {
-	remainingCreated   map[string]bool
-	remainingReady     map[string]bool
-	addressSpace       *enmassev1beta1.AddressSpace
-	adapterCredentials *iotv1alpha1.Credentials
+	remainingCreated map[string]bool
+	remainingReady   map[string]bool
+	addressSpace     *enmassev1beta1.AddressSpace
 }
 
 func updateFromMap(resources map[string]bool, condition *iotv1alpha1.CommonCondition, reason string) {
@@ -75,7 +86,7 @@ func updateManagedStatus(m *managedStatus, project *iotv1alpha1.IoTProject) {
 
 func (r *ReconcileIoTProject) reconcileManaged(ctx context.Context, request *reconcile.Request, project *iotv1alpha1.IoTProject) (reconcile.Result, error) {
 
-	log.Info("Reconcile project with managed strategy")
+	now := time.Now()
 
 	strategy := project.Spec.DownstreamStrategy.ManagedDownstreamStrategy
 
@@ -90,6 +101,7 @@ func (r *ReconcileIoTProject) reconcileManaged(ctx context.Context, request *rec
 	managedStatus.remainingReady[resourceTypeAdapterUser] = false
 
 	managedStatus.remainingCreated[resourceTypeAddressSpace] = false
+	managedStatus.remainingReady[resourceTypeAdapterUser] = false
 
 	// defer condition status update
 
@@ -99,6 +111,19 @@ func (r *ReconcileIoTProject) reconcileManaged(ctx context.Context, request *rec
 
 	rc := recon.ReconcileContext{}
 
+	// ensure adapter credentials are set
+
+	rc.Process(func() (result reconcile.Result, e error) {
+		return r.ensureAdapterCredentials(ctx, project)
+	})
+
+	if rc.NeedRequeue() || rc.Error() != nil {
+		// return early as we need to be persisted
+		return rc.Result()
+	}
+
+	project.Status.Phase = "Configuring"
+
 	// reconcile address space
 
 	rc.Process(func() (result reconcile.Result, e error) {
@@ -116,34 +141,144 @@ func (r *ReconcileIoTProject) reconcileManaged(ctx context.Context, request *rec
 	// create a new user for protocol adapters
 
 	rc.Process(func() (result reconcile.Result, e error) {
-		credentials, err := r.reconcileAdapterUser(ctx, project, strategy, managedStatus)
-		managedStatus.adapterCredentials = credentials
-		return reconcile.Result{}, err
+		return r.reconcileAdapterUser(ctx, project, strategy, managedStatus)
 	})
 
 	// extract endpoint information
 
-	project.Status.DownstreamEndpoint = nil
-	if managedStatus.addressSpace != nil && managedStatus.adapterCredentials != nil {
+	currentCredentials := project.Status.DownstreamEndpoint.Credentials.DeepCopy()
+	if managedStatus.addressSpace != nil && project.Status.IsReady {
 
 		forceTls := true
 		rc.Process(func() (result reconcile.Result, e error) {
-			endpoint, err := extractEndpointInformation("messaging", iotv1alpha1.Service, "amqps", managedStatus.adapterCredentials, managedStatus.addressSpace, &forceTls)
+			endpoint, err := extractEndpointInformation("messaging", iotv1alpha1.Service, "amqps", currentCredentials, managedStatus.addressSpace, &forceTls)
 
 			if endpoint != nil {
-				project.Status.DownstreamEndpoint = endpoint.DeepCopy()
+				project.Status.DownstreamEndpoint = endpoint.ConnectionInformation.DeepCopy()
 			}
 
 			return reconcile.Result{}, err
 		})
 	}
 
+	// check and queue password reset
+
+	rc.Process(func() (result reconcile.Result, e error) {
+
+		if val, ok := project.Annotations[annotationBase+"/resetPasswordAfter"]; ok {
+			expires, err := time.Parse(time.RFC3339, val)
+			if err != nil {
+				return reconcile.Result{}, err
+			}
+			delay := expires.Sub(now)
+			if delay.Seconds() > 0 {
+				log.Info("Password set to expire in", "delay", delay)
+				return reconcile.Result{RequeueAfter: delay}, nil
+			}
+		}
+		return reconcile.Result{}, nil
+	})
+
 	// return result
 
 	return rc.Result()
 
 }
 
+func (r *ReconcileIoTProject) ensureAdapterCredentials(ctx context.Context, project *iotv1alpha1.IoTProject) (reconcile.Result, error) {
+
+	if project.Status.DownstreamEndpoint == nil {
+		project.Status.DownstreamEndpoint = &iotv1alpha1.ConnectionInformation{}
+	}
+
+	if project.Status.Managed == nil {
+		project.Status.Managed = &iotv1alpha1.ManagedStatus{}
+	}
+
+	//
+
+	changed := false
+
+	// eval address space
+
+	if project.Status.Managed.AddressSpace != project.Spec.DownstreamStrategy.ManagedDownstreamStrategy.AddressSpace.Name {
+
+		if project.Status.Managed.AddressSpace == "" {
+
+			// record change, early store and requeue
+
+			project.Status.Managed.AddressSpace = project.Spec.DownstreamStrategy.ManagedDownstreamStrategy.AddressSpace.Name
+			changed = true
+
+		} else {
+
+			// cleanup old address space first
+
+			project.Status.Phase = "Reconfiguring"
+			project.Status.PhaseReason = "Address Space changed"
+
+			result, err := cleanupManagedResources(ctx, r.client, project)
+			if err != nil {
+				// failed to clean up ... return
+				return result, err
+			}
+
+			// clear out address space, will re-set in the next iteration
+
+			project.Status.Managed.AddressSpace = ""
+			changed = true
+
+		}
+
+	}
+
+	// eval password
+
+	if val, ok := project.Annotations[annotationBase+"/resetPasswordAfter"]; ok {
+		expires, err := time.Parse(time.RFC3339, val)
+		if err != nil {
+			return reconcile.Result{}, err
+		}
+		if time.Now().After(expires) && expires.After(project.Status.Managed.PasswordTime.Time) {
+			log.Info("Password expired")
+			// reset password
+			project.Status.DownstreamEndpoint.Password = ""
+		}
+	}
+
+	if project.Status.DownstreamEndpoint.Password == "" || project.Status.Managed.PasswordTime.Time.IsZero() {
+
+		project.Status.Managed.PasswordTime.Time = time.Now()
+
+		// ... we generate a new one ...
+		log.Info("Generating new password")
+		password, err := util.GeneratePassword(32)
+
+		if err != nil {
+			return reconcile.Result{}, err
+		}
+
+		project.Status.DownstreamEndpoint = &iotv1alpha1.ConnectionInformation{
+			Credentials: iotv1alpha1.Credentials{
+				Username: "adapter-" + string(project.UID),
+				Password: password,
+			},
+		}
+
+		// re-queue right now to ensure the password is stored
+
+		changed = true
+
+	}
+
+	// proceed
+
+	return reconcile.Result{
+		Requeue: changed,
+	}, nil
+
+}
+
 func (r *ReconcileIoTProject) reconcileAddressSpace(ctx context.Context, project *iotv1alpha1.IoTProject, strategy *iotv1alpha1.ManagedDownstreamStrategy, managedStatus *managedStatus) (*enmassev1beta1.AddressSpace, reconcile.Result, error) {
 
 	addressSpace := &enmassev1beta1.AddressSpace{
@@ -158,10 +293,6 @@ func (r *ReconcileIoTProject) reconcileAddressSpace(ctx context.Context, project
 	_, err := controllerutil.CreateOrUpdate(ctx, r.client, addressSpace, func(existing runtime.Object) error {
 		existingAddressSpace := existing.(*enmassev1beta1.AddressSpace)
 
-		if err := r.ensureOwnerIsSet(project, existingAddressSpace); err != nil {
-			return err
-		}
-
 		log.V(2).Info("Reconcile address space", "AddressSpace", existingAddressSpace)
 
 		managedStatus.remainingReady[resourceTypeAddressSpace] = existingAddressSpace.Status.IsReady
@@ -183,11 +314,13 @@ func (r *ReconcileIoTProject) reconcileAddressSpace(ctx context.Context, project
 
 }
 
-func (r *ReconcileIoTProject) reconcileAdapterUser(ctx context.Context, project *iotv1alpha1.IoTProject, strategy *iotv1alpha1.ManagedDownstreamStrategy, managedStatus *managedStatus) (*iotv1alpha1.Credentials, error) {
+func (r *ReconcileIoTProject) reconcileAdapterUser(ctx context.Context, project *iotv1alpha1.IoTProject, strategy *iotv1alpha1.ManagedDownstreamStrategy, managedStatus *managedStatus) (reconcile.Result, error) {
 
-	credentials := &iotv1alpha1.Credentials{
-		Username: "adapter",
-	}
+	// ensured that this is not nil
+
+	credentials := project.Status.DownstreamEndpoint.Credentials
+
+	// construct object
 
 	adapterUser := &userv1beta1.MessagingUser{
 		ObjectMeta: metav1.ObjectMeta{
@@ -209,7 +342,7 @@ func (r *ReconcileIoTProject) reconcileAdapterUser(ctx context.Context, project
 		managedStatus.remainingReady[resourceTypeAdapterUser] = true
 	}
 
-	return credentials, err
+	return reconcile.Result{}, err
 }
 
 func (r *ReconcileIoTProject) reconcileAddress(project *iotv1alpha1.IoTProject, strategy *iotv1alpha1.ManagedDownstreamStrategy, addressName string, plan string, typeName string, existing *enmassev1beta1.Address) error {
@@ -313,51 +446,89 @@ func (r *ReconcileIoTProject) reconcileAddressSet(ctx context.Context, project *
 
 func (r *ReconcileIoTProject) reconcileManagedAddressSpace(project *iotv1alpha1.IoTProject, strategy *iotv1alpha1.ManagedDownstreamStrategy, existing *enmassev1beta1.AddressSpace) error {
 
-	if existing.CreationTimestamp.IsZero() {
-		existing.ObjectMeta.Labels = project.Labels
+	// add ourselves to the list of owners
+
+	if err := r.ensureOwnerIsSet(project, existing); err != nil {
+		return err
 	}
 
+	// we must have a plan
+
 	if strategy.AddressSpace.Plan == "" {
 		return fmt.Errorf("'addressSpace.plan' must not be empty")
 	}
 
-	existing.Spec.Type = StringOrDefault(strategy.AddressSpace.Type, "standard")
-	existing.Spec.Plan = strategy.AddressSpace.Plan
+	// eval type information
+
+	t := StringOrDefault(strategy.AddressSpace.Type, "standard")
+
+	// check if we are the only one using this space
+
+	if len(existing.OwnerReferences) == 1 {
+
+		// if so, we simply set the information
+
+		existing.Spec.Type = t
+		existing.Spec.Plan = strategy.AddressSpace.Plan
+
+	} else {
+
+		// otherwise, we must ensure we have the same setting
+
+		if existing.Spec.Type != t {
+			return fmt.Errorf("address space is already created using a different type: %s", existing.Spec.Type)
+		}
+		if existing.Spec.Plan != strategy.AddressSpace.Plan {
+			return fmt.Errorf("address space is already created using a different plan: %s", existing.Spec.Plan)
+		}
+
+	}
 
 	return nil
 }
 
-func (r *ReconcileIoTProject) reconcileAdapterMessagingUser(project *iotv1alpha1.IoTProject, credentials *iotv1alpha1.Credentials, existing *userv1beta1.MessagingUser) error {
+func (r *ReconcileIoTProject) reconcileAdapterMessagingUser(project *iotv1alpha1.IoTProject, credentials iotv1alpha1.Credentials, existing *userv1beta1.MessagingUser) error {
 
 	if err := r.ensureControllerOwnerIsSet(project, existing); err != nil {
 		return err
 	}
 
+	managed := project.Status.Managed
+
 	existing.Spec.Username = credentials.Username
 	existing.Spec.Authentication = userv1beta1.AuthenticationSpec{
 		Type: "password",
 	}
 
-	// if the status has endpoint information ...
-	if project.Status.DownstreamEndpoint != nil {
-		// ... we extract the password and return it
-		credentials.Password = project.Status.DownstreamEndpoint.Password
+	if existing.Annotations == nil {
+		existing.Annotations = make(map[string]string)
 	}
 
-	// if the project is ready and password is not set yet ...
-	if (project.Status.IsReady && credentials.Password == "") || util.IsNewObject(existing) {
-		// ... we generate a new one ...
-		log.Info("Generating new password")
-		password, err := util.GeneratePassword(32)
-		if err != nil {
-			return err
-		}
-		// ... return it in the credentials structure
-		credentials.Password = password
-		// ... and set it to the messaging user
+	existing.Annotations[annotationProject] = project.Name
+	existing.Annotations[annotationProjectUID] = string(project.UID)
+
+	// get last password sync
+
+	var syncTime time.Time
+	syncTimeString := existing.Annotations[annotationPasswordSyncTime]
+	if syncTimeString != "" {
+		syncTime, _ = time.Parse(time.RFC3339, syncTimeString)
+	}
+
+	// if we missed an update
+
+	if managed.PasswordTime.After(syncTime) {
+
+		log.Info("Updating password for", "user", existing.Name)
+
+		// set the password
 		existing.Spec.Authentication.Password = []byte(credentials.Password)
+		existing.Annotations[annotationPasswordSyncTime] = managed.PasswordTime.UTC().Format(time.RFC3339)
+
 	}
 
+	// create access rules
+
 	telemetryName := util.AddressName(project, AddressNameTelemetry)
 	eventName := util.AddressName(project, AddressNameEvent)
 
diff --git a/pkg/controller/iotproject/provided.go b/pkg/controller/iotproject/provided.go
index fcb2e05d4..23f4a9b91 100644
--- a/pkg/controller/iotproject/provided.go
+++ b/pkg/controller/iotproject/provided.go
@@ -29,7 +29,7 @@ func (r *ReconcileIoTProject) reconcileProvided(ctx context.Context, request *re
 
 	project.Status.DownstreamEndpoint = nil
 	if endpoint != nil {
-		project.Status.DownstreamEndpoint = endpoint.DeepCopy()
+		project.Status.DownstreamEndpoint = endpoint.ConnectionInformation.DeepCopy()
 	}
 
 	return reconcile.Result{}, err
diff --git a/pkg/controller/iotproject/utils.go b/pkg/controller/iotproject/utils.go
index 59d44a671..4d23aea1c 100644
--- a/pkg/controller/iotproject/utils.go
+++ b/pkg/controller/iotproject/utils.go
@@ -17,6 +17,8 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 )
 
+const annotationBase = "iot.enmasse.io"
+
 // Ensure that a controller owner is set
 // As there may only be one, we only to this when the creation timestamp is zero
 func (r *ReconcileIoTProject) ensureControllerOwnerIsSet(owner, object v1.Object) error {
diff --git a/pkg/util/address.go b/pkg/util/address.go
index 8940ce8cb..f96111700 100644
--- a/pkg/util/address.go
+++ b/pkg/util/address.go
@@ -7,6 +7,7 @@ package util
 
 import (
 	"regexp"
+	"strings"
 
 	"github.com/google/uuid"
 
@@ -14,10 +15,10 @@ import (
 )
 
 var (
-	addressNameExpression  *regexp.Regexp = regexp.MustCompile("^[a-zA-Z]+$")
-	replaceExpression      *regexp.Regexp = regexp.MustCompile("[^a-zA-Z]")
-	replaceStartExpression *regexp.Regexp = regexp.MustCompile("^[^a-zA-Z]+")
-	operatorUuidNamespace  uuid.UUID      = uuid.MustParse("1516b246-23aa-11e9-b615-c85b762e5a2c")
+	addressNameExpression *regexp.Regexp = regexp.MustCompile("^[a-z0-9\\-]+$")
+	replaceExpression     *regexp.Regexp = regexp.MustCompile("[^a-z0-9\\-]")
+	replaceEndExpression  *regexp.Regexp = regexp.MustCompile("[^a-z0-9]+$")
+	operatorUuidNamespace uuid.UUID      = uuid.MustParse("1516b246-23aa-11e9-b615-c85b762e5a2c")
 )
 
 // Get an address name from an IoTProject
@@ -32,25 +33,28 @@ func EncodeAsMetaName(name string, maxLength int) string {
 		return name
 	}
 
-	newPrefix := replaceExpression.ReplaceAllString(name, "")
-	if len(newPrefix) > 0 {
-		newPrefix = newPrefix + "-"
-	}
+	name = strings.ToLower(name)
+
+	prefix := replaceExpression.ReplaceAllString(name, "")
 
-	name = newPrefix + uuid.NewMD5(operatorUuidNamespace, []byte(name)).String()
+	id := uuid.NewMD5(operatorUuidNamespace, []byte(name)).String()
 
 	if maxLength > 0 {
-		rname := []rune(name)
-		l := len(rname)
+		rprefix := []rune(prefix)
+		idlen := 1 + len(id)
+		l := len(rprefix) + idlen
 		if l > maxLength {
-			s := l - maxLength
-			rname = rname[s:l]
-			name = string(rname)
-			name = replaceStartExpression.ReplaceAllString(name, "")
+			rprefix = rprefix[:maxLength-idlen]
+			prefix = string(rprefix)
+			prefix = replaceEndExpression.ReplaceAllString(prefix, "")
 		}
 	}
 
-	return name
+	if len(prefix) > 0 {
+		prefix = prefix + "-"
+	}
+
+	return prefix + id
 }
 
 // Encode an address name so that it can be put inside the .metadata.name field of an Address object
diff --git a/pkg/util/install/delete.go b/pkg/util/install/delete.go
index 5e49a7002..58cbec98b 100644
--- a/pkg/util/install/delete.go
+++ b/pkg/util/install/delete.go
@@ -223,8 +223,7 @@ func BulkRemoveOwner(ctx context.Context, c client.Client, owner runtime.Object,
 
 		// process ownership
 
-		result, err := ProcessOwnedByObject(owner, i, controller, true)
-
+		result, err := RemoveAsOwner(owner, i, controller)
 		if err != nil {
 			return nil, err
 		}
@@ -247,3 +246,7 @@ func BulkRemoveOwner(ctx context.Context, c client.Client, owner runtime.Object,
 	})
 
 }
+
+func RemoveAsOwner(owner runtime.Object, object interface{}, controller bool) (OwnerResult, error) {
+	return ProcessOwnedByObject(owner, object, controller, true)
+}
diff --git a/templates/csv/enmasse.clusterserviceversion.yaml b/templates/csv/enmasse.clusterserviceversion.yaml
index d88314bce..0eb60caf1 100644
--- a/templates/csv/enmasse.clusterserviceversion.yaml
+++ b/templates/csv/enmasse.clusterserviceversion.yaml
@@ -1253,3 +1253,14 @@ spec:
             path: conditions
             x-descriptors:
               - 'urn:alm:descriptor:io.kubernetes.conditions'
+
+          - description: The phase of the project.
+            displayName: Phase
+            path: phase
+            x-descriptors:
+              - 'urn:alm:descriptor:io.kubernetes.phase'
+          - description: Description of the project phase.
+            displayName: Reason
+            path: phaseReason
+            x-descriptors:
+              - 'urn:alm:descriptor:io.kubernetes.phase:reason'
diff --git a/templates/iot/api/010-CustomResourceDefinition-iot-project.yaml b/templates/iot/api/010-CustomResourceDefinition-iot-project.yaml
index 6183b3972..7109cf074 100644
--- a/templates/iot/api/010-CustomResourceDefinition-iot-project.yaml
+++ b/templates/iot/api/010-CustomResourceDefinition-iot-project.yaml
@@ -32,9 +32,9 @@ spec:
     type: boolean
     description: If TLS is enabled
     JSONPath: .status.downstreamEndpoint.tls
-  - name: Ready
-    type: boolean
-    description: If the IoT project is ready
-    JSONPath: .status.isReady
+  - name: Phase
+    type: string
+    description: Phase of the IoT project
+    JSONPath: .status.phase
   subresources:
     status: {}
diff --git a/templates/iot/examples/infinispan/manual/030-Service.yaml b/templates/iot/examples/infinispan/manual/030-Service.yaml
index c216a27a6..a667fa144 100644
--- a/templates/iot/examples/infinispan/manual/030-Service.yaml
+++ b/templates/iot/examples/infinispan/manual/030-Service.yaml
@@ -3,7 +3,7 @@ kind: Service
 metadata:
   labels:
     app: infinispan
-  name: infinispan-server
+  name: infinispan # align name with operator version
 spec:
   ports:
   - port: 11222
@@ -13,5 +13,5 @@ spec:
     targetPort: 9990
     name: management
   selector:
-    name: infinispan-server
+    name: infinispan
     app: infinispan
\ No newline at end of file
diff --git a/templates/iot/examples/infinispan/manual/050-StatefulSet.yaml b/templates/iot/examples/infinispan/manual/050-StatefulSet.yaml
index 371d59f53..1d4f0155c 100644
--- a/templates/iot/examples/infinispan/manual/050-StatefulSet.yaml
+++ b/templates/iot/examples/infinispan/manual/050-StatefulSet.yaml
@@ -2,22 +2,22 @@ kind: StatefulSet
 apiVersion: apps/v1beta1
 metadata:
   labels:
-    name: infinispan-server
+    name: infinispan
     app: infinispan
-  name: infinispan-server
+  name: infinispan
 spec:
   replicas: 1
   serviceName: infinispan-server-headless
   selector:
     matchLabels:
-      name: infinispan-server
+      name: infinispan
       app: infinispan
   template:
     metadata:
       labels:
-        name: infinispan-server
+        name: infinispan
         app: infinispan
-      name: infinispan-server
+      name: infinispan
     spec:
       serviceAccount: infinispan
       containers:
@@ -26,7 +26,7 @@ spec:
         - -Djboss.default.jgroups.stack=kubernetes
         env:
         - name: KUBERNETES_LABELS
-          value: app=infinispan,name=infinispan-server
+          value: app=infinispan,name=infinispan
         - name: KUBERNETES_NAMESPACE
           valueFrom:
             fieldRef:


Test patch:
diff --git a/api-model/src/test/java/io/enmasse/address/model/v1/address/KubeUtilTest.java b/api-model/src/test/java/io/enmasse/address/model/v1/address/KubeUtilTest.java
index 08d87112e..16751717b 100644
--- a/api-model/src/test/java/io/enmasse/address/model/v1/address/KubeUtilTest.java
+++ b/api-model/src/test/java/io/enmasse/address/model/v1/address/KubeUtilTest.java
@@ -291,4 +291,13 @@ public class KubeUtilTest {
     public void testSanitizeUuidName(final String name, final String uuid, final String output) {
         assertEquals(output, KubeUtil.sanitizeWithUuid(name, uuid));
     }
+
+    /**
+     * Test for encoding with Go logic.
+     */
+    @Test
+    public void testSanitizeForGo () {
+        assertEquals("as1.telemetryiot-project-ns-4cf002ae-37a5-38d6-9749-b7f85b29a385", KubeUtil.sanitizeForGo("as1", "telemetry/iot-project-ns.iot1"));
+    }
+
 }
diff --git a/api-model/src/test/java/io/enmasse/user/model/v1/UserModelTest.java b/api-model/src/test/java/io/enmasse/user/model/v1/UserModelTest.java
index 5857f5cbd..c2521d38d 100644
--- a/api-model/src/test/java/io/enmasse/user/model/v1/UserModelTest.java
+++ b/api-model/src/test/java/io/enmasse/user/model/v1/UserModelTest.java
@@ -24,6 +24,7 @@ public class UserModelTest {
                 .withMetadata(new ObjectMetaBuilder()
                         .withName("myspace.user1")
                         .withNamespace("ns1")
+                        .addToAnnotations("foo", "bar")
                         .build())
                 .withSpec(new UserSpecBuilder()
                         .withUsername("user1")
@@ -60,15 +61,20 @@ public class UserModelTest {
         UserList list = new UserList();
         list.getItems().add(user);
 
+        // serialize & deserialize
+
         serialized = mapper.writeValueAsBytes(list);
         UserList deserializedList = mapper.readValue(serialized, UserList.class);
 
+        // assert deserialized
+
         assertEquals(1, deserializedList.getItems().size());
 
         deserialized = deserializedList.getItems().get(0);
 
         assertEquals(user.getMetadata().getName(), deserialized.getMetadata().getName());
         assertEquals(user.getMetadata().getNamespace(), deserialized.getMetadata().getNamespace());
+        assertEquals("bar", deserialized.getMetadata().getAnnotations().get("foo"));
         assertEquals(user.getSpec().getUsername(), deserialized.getSpec().getUsername());
         assertEquals(user.getSpec().getAuthentication().getType(), deserialized.getSpec().getAuthentication().getType());
         assertEquals(user.getSpec().getAuthentication().getPassword(), deserialized.getSpec().getAuthentication().getPassword());
diff --git a/keycloak-user-api/src/test/java/io/enmasse/user/keycloak/KeycloakUserApiTest.java b/keycloak-user-api/src/test/java/io/enmasse/user/keycloak/KeycloakUserApiTest.java
index 4fda9f260..d05def7b7 100644
--- a/keycloak-user-api/src/test/java/io/enmasse/user/keycloak/KeycloakUserApiTest.java
+++ b/keycloak-user-api/src/test/java/io/enmasse/user/keycloak/KeycloakUserApiTest.java
@@ -11,6 +11,7 @@ import static java.util.stream.Collectors.toList;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -313,7 +314,7 @@ public class KeycloakUserApiTest {
      * Assert groups. <br>
      * A convenience method to assert group information. Compares the groups as
      * sorted arrays.
-     * 
+     *
      * @param actual   The actual result.
      * @param expected The expected result.
      */
@@ -351,4 +352,18 @@ public class KeycloakUserApiTest {
         assertTrue(KeycloakUserApi.hasAttribute(realm, "foo", null));
         assertFalse(KeycloakUserApi.hasAttribute(realm, null, null));
     }
+
+    @Test
+    public void testAnnotations() {
+        final Map<String,String> annotations = new HashMap<>();
+        annotations.put("foo", "bar");
+        annotations.put("bar", "baz");
+
+        final List<String> encoded = KeycloakUserApi.annotationsToString(annotations);
+        assertNotNull(encoded);
+        assertEquals(2,encoded.size());
+
+        final Map<String, String> actualAnnotations = KeycloakUserApi.annotationsFromString(encoded);
+        assertEquals(annotations, actualAnnotations);
+    }
 }
diff --git a/pkg/util/address_test.go b/pkg/util/address_test.go
new file mode 100644
index 000000000..96b85fe52
--- /dev/null
+++ b/pkg/util/address_test.go
@@ -0,0 +1,28 @@
+/*
+ * Copyright 2019, EnMasse authors.
+ * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html).
+ */
+
+package util
+
+import (
+	"testing"
+)
+
+func TestAddressName(t *testing.T) {
+	for _, v := range []struct {
+		addressSpaceName string
+		addressName      string
+		expected         string
+	}{
+		{addressSpaceName: "as1", addressName: "telemetry/iot-project-ns.iot1", expected: "as1.telemetryiot-project-ns-4cf002ae-37a5-38d6-9749-b7f85b29a385"},
+		{addressSpaceName: "as1", addressName: "event/iot-project-ns.iot1", expected: "as1.eventiot-project-nsiot1-82203f07-4081-392b-a834-dd1ffcaa2c5f"},
+	} {
+
+		out := EncodeAddressSpaceAsMetaName(v.addressSpaceName, v.addressName)
+		if out != v.expected {
+			t.Errorf("Encoding error - input: '%s', '%s' expected: '%s', actual: '%s'", v.addressSpaceName, v.addressName, v.expected, out)
+		}
+
+	}
+}
diff --git a/systemtests/src/main/java/io/enmasse/systemtest/Environment.java b/systemtests/src/main/java/io/enmasse/systemtest/Environment.java
index c66f8c5db..12424d2ee 100644
--- a/systemtests/src/main/java/io/enmasse/systemtest/Environment.java
+++ b/systemtests/src/main/java/io/enmasse/systemtest/Environment.java
@@ -33,6 +33,7 @@ public class Environment {
     public static final String IS_OCP4_REGEXP = "4.*";
     private static final String DEFAULT_DEVICE_REGISTRY = "DEFAULT_DEVICE_REGISTRY";
     private static final String SKIP_SAVE_STATE = "SKIP_SAVE_STATE";
+    private static final String SKIP_DEPLOY_INFINISPAN = "SKIP_DEPLOY_INFINISPAN";
     private static Logger log = CustomLogger.getLogger();
     private static Environment instance;
     private final String namespace = System.getenv(K8S_NAMESPACE_ENV);
@@ -53,6 +54,7 @@ public class Environment {
     private final String appName = System.getenv().getOrDefault(APP_NAME_ENV, "enmasse");
     private final String defaultDeviceRegistry = System.getenv().getOrDefault(DEFAULT_DEVICE_REGISTRY, "file");
     private final boolean skipSaveState = Boolean.parseBoolean(System.getenv(SKIP_SAVE_STATE));
+    private final boolean skipDeployInfinispan = Boolean.parseBoolean(System.getenv(SKIP_DEPLOY_INFINISPAN));
     protected UserCredentials managementCredentials = new UserCredentials(null, null);
     protected UserCredentials defaultCredentials = new UserCredentials(null, null);
 
@@ -190,9 +192,15 @@ public class Environment {
     public void setDefaultCredentials(UserCredentials defaultCredentials) {
         this.defaultCredentials = defaultCredentials;
     }
+
     public boolean isSkipSaveState() {
-        return skipSaveState;
+        return this.skipSaveState;
+    }
+
+    public boolean isSkipDeployInfinispan() {
+        return this.skipDeployInfinispan;
     }
+
     public String getDefaultDeviceRegistry() {
         return defaultDeviceRegistry;
     }
diff --git a/systemtests/src/main/java/io/enmasse/systemtest/platform/Kubernetes.java b/systemtests/src/main/java/io/enmasse/systemtest/platform/Kubernetes.java
index eba82b3bc..2a50a1c91 100644
--- a/systemtests/src/main/java/io/enmasse/systemtest/platform/Kubernetes.java
+++ b/systemtests/src/main/java/io/enmasse/systemtest/platform/Kubernetes.java
@@ -88,6 +88,7 @@ import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeUnit;
@@ -275,6 +276,7 @@ public abstract class Kubernetes {
 
     public Endpoint getEndpoint(String serviceName, String namespace, String port) {
         Service service = client.services().inNamespace(namespace).withName(serviceName).get();
+        Objects.requireNonNull(service, () -> String.format("Unable to find service '%s' in namespace '%s'", serviceName, namespace));
         return new Endpoint(service.getSpec().getClusterIP(), getPort(service, port));
     }
 
diff --git a/systemtests/src/main/java/io/enmasse/systemtest/platform/apps/SystemtestsKubernetesApps.java b/systemtests/src/main/java/io/enmasse/systemtest/platform/apps/SystemtestsKubernetesApps.java
index eeb4b32e5..25669dfae 100644
--- a/systemtests/src/main/java/io/enmasse/systemtest/platform/apps/SystemtestsKubernetesApps.java
+++ b/systemtests/src/main/java/io/enmasse/systemtest/platform/apps/SystemtestsKubernetesApps.java
@@ -61,7 +61,7 @@ public class SystemtestsKubernetesApps {
     public static final String SELENIUM_CONFIG_MAP = "rhea-configmap";
     public static final String OPENSHIFT_CERT_VALIDATOR = "systemtests-cert-validator";
     public static final String POSTGRES_APP = "postgres-app";
-    public static final String INFINISPAN_SERVER = "infinispan-server";
+    public static final String INFINISPAN_SERVER = "infinispan";
     private static final Path INFINISPAN_EXAMPLE_BASE = Paths.get("../templates/iot/examples/infinispan");
 
     public static String getMessagingAppPodName() throws Exception {
@@ -196,7 +196,11 @@ public class SystemtestsKubernetesApps {
         return in -> ReplaceValueStream.replaceValues(in, values);
     }
 
-    public static Endpoint deployInfinispanServer(String namespace) throws Exception {
+    public static Endpoint deployInfinispanServer(final String namespace) throws Exception {
+
+        if (Environment.getInstance().isSkipDeployInfinispan()) {
+            return getInfinispanEndpoint(namespace);
+        }
 
         final Kubernetes kubeCli = Kubernetes.getInstance();
         final KubernetesClient client = kubeCli.getClient();
@@ -217,17 +221,25 @@ public class SystemtestsKubernetesApps {
 
         // return hotrod enpoint
 
-        return kubeCli.getEndpoint(INFINISPAN_SERVER, namespace, "hotrod");
+        return getInfinispanEndpoint(namespace);
     }
 
+    private static Endpoint getInfinispanEndpoint(final String namespace) {
+        return Kubernetes.getInstance().getEndpoint(INFINISPAN_SERVER, namespace, "hotrod");
+    }
 
     public static void deleteInfinispanServer(final String namespace) throws Exception {
 
+        if (Environment.getInstance().isSkipDeployInfinispan()) {
+            return;
+        }
+
         // delete "common" and "manual" folders
 
         deleteDirectories(namespaceReplacer(namespace),
                 INFINISPAN_EXAMPLE_BASE.resolve("common"),
                 INFINISPAN_EXAMPLE_BASE.resolve("manual"));
+
     }
 
     public static void deployAMQBroker(String namespace, String name, String user, String password, BrokerCertBundle certBundle) throws Exception {
diff --git a/systemtests/src/main/java/io/enmasse/systemtest/time/TimeoutBudget.java b/systemtests/src/main/java/io/enmasse/systemtest/time/TimeoutBudget.java
index 5768a62c7..b83648819 100644
--- a/systemtests/src/main/java/io/enmasse/systemtest/time/TimeoutBudget.java
+++ b/systemtests/src/main/java/io/enmasse/systemtest/time/TimeoutBudget.java
@@ -41,6 +41,14 @@ public class TimeoutBudget {
         }
     }
 
+    /**
+     * Get the remaining time of the budget.
+     * @return The remaining time.
+     */
+    public Duration remaining() {
+        return Duration.ofMillis(timeLeft());
+    }
+
     public boolean timeoutExpired() {
         return timeLeft() < 0;
     }
diff --git a/systemtests/src/main/java/io/enmasse/systemtest/utils/AddressSpaceUtils.java b/systemtests/src/main/java/io/enmasse/systemtest/utils/AddressSpaceUtils.java
index 2b45e4240..f16408af7 100644
--- a/systemtests/src/main/java/io/enmasse/systemtest/utils/AddressSpaceUtils.java
+++ b/systemtests/src/main/java/io/enmasse/systemtest/utils/AddressSpaceUtils.java
@@ -29,6 +29,7 @@ import java.util.Optional;
 import java.util.concurrent.Callable;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
+import java.util.function.BooleanSupplier;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -58,6 +59,10 @@ public class AddressSpaceUtils {
                 .map(addressSpace -> addressSpace.getMetadata().getName()).collect(Collectors.toList()).contains(addressSpaceName);
     }
 
+    public static BooleanSupplier addressSpaceExists(final String namespace, final String name) {
+        return () -> existAddressSpace(namespace, name);
+    }
+
     public static boolean isAddressSpaceReady(AddressSpace addressSpace) {
         return addressSpace != null && addressSpace.getStatus().isReady();
     }
diff --git a/systemtests/src/main/java/io/enmasse/systemtest/utils/AddressUtils.java b/systemtests/src/main/java/io/enmasse/systemtest/utils/AddressUtils.java
index ded30fe49..07c069b71 100644
--- a/systemtests/src/main/java/io/enmasse/systemtest/utils/AddressUtils.java
+++ b/systemtests/src/main/java/io/enmasse/systemtest/utils/AddressUtils.java
@@ -191,28 +191,24 @@ public class AddressUtils {
 
     public static void waitForDestinationsReady(TimeoutBudget budget, Address... destinations) throws Exception {
         String operationID = TimeMeasuringSystem.startOperation(SystemtestsOperation.ADDRESS_WAIT_READY);
-        Thread.sleep(2000);
         waitForAddressesMatched(budget, destinations.length, getAddressClient(destinations), addressList -> checkAddressesMatching(addressList, AddressUtils::isAddressReady, destinations));
         TimeMeasuringSystem.stopOperation(operationID);
     }
 
     public static void waitForDestinationPlanApplied(TimeoutBudget budget, Address... destinations) throws Exception {
         String operationID = TimeMeasuringSystem.startOperation(SystemtestsOperation.ADDRESS_WAIT_PLAN_CHANGE);
-        Thread.sleep(2000);
         waitForAddressesMatched(budget, destinations.length, getAddressClient(destinations), addressList -> checkAddressesMatching(addressList, AddressUtils::isPlanSynced, destinations));
         TimeMeasuringSystem.stopOperation(operationID);
     }
 
     public static void waitForBrokersDrained(TimeoutBudget budget, Address... destinations) throws Exception {
         String operationID = TimeMeasuringSystem.startOperation(SystemtestsOperation.ADDRESS_WAIT_BROKER_DRAINED);
-        Thread.sleep(2000);
         waitForAddressesMatched(budget, destinations.length, getAddressClient(destinations), addressList -> checkAddressesMatching(addressList, AddressUtils::areBrokersDrained, destinations));
         TimeMeasuringSystem.stopOperation(operationID);
     }
 
     public static void waitForForwardersReady(TimeoutBudget budget, Address... destinations) throws Exception {
         String operationID = TimeMeasuringSystem.startOperation(SystemtestsOperation.ADDRESS_WAIT_FORWARDERS);
-        Thread.sleep(2000);
         waitForAddressesMatched(budget, destinations.length, getAddressClient(destinations), addressList -> checkAddressesMatching(addressList, AddressUtils::areForwardersReady, destinations));
         TimeMeasuringSystem.stopOperation(operationID);
     }
diff --git a/systemtests/src/main/java/io/enmasse/systemtest/utils/IoTUtils.java b/systemtests/src/main/java/io/enmasse/systemtest/utils/IoTUtils.java
index 2edf525cf..ab2cf8b65 100644
--- a/systemtests/src/main/java/io/enmasse/systemtest/utils/IoTUtils.java
+++ b/systemtests/src/main/java/io/enmasse/systemtest/utils/IoTUtils.java
@@ -103,7 +103,9 @@ public class IoTUtils {
                 && project.getSpec().getDownstreamStrategy().getManagedStrategy().getAddressSpace().getName() != null
         ) {
             var addressSpaceName = project.getSpec().getDownstreamStrategy().getManagedStrategy().getAddressSpace().getName();
-            AddressSpaceUtils.waitForAddressSpaceReady(Kubernetes.getInstance().getAddressSpaceClient(project.getMetadata().getNamespace()).withName(addressSpaceName).get(), budget);
+            var addressSpace = Kubernetes.getInstance().getAddressSpaceClient(project.getMetadata().getNamespace()).withName(addressSpaceName).get();
+            Objects.requireNonNull(addressSpace, () -> String.format("Unable to find addressSpace: %s", addressSpaceName));
+            AddressSpaceUtils.waitForAddressSpaceReady(addressSpace, budget);
         }
     }
 
diff --git a/systemtests/src/main/java/io/enmasse/systemtest/utils/TestUtils.java b/systemtests/src/main/java/io/enmasse/systemtest/utils/TestUtils.java
index 09e42aaa6..958dee896 100644
--- a/systemtests/src/main/java/io/enmasse/systemtest/utils/TestUtils.java
+++ b/systemtests/src/main/java/io/enmasse/systemtest/utils/TestUtils.java
@@ -32,12 +32,15 @@ import org.openqa.selenium.Capabilities;
 import org.openqa.selenium.chrome.ChromeOptions;
 import org.openqa.selenium.firefox.FirefoxOptions;
 import org.openqa.selenium.remote.RemoteWebDriver;
+import org.opentest4j.AssertionFailedError;
 import org.slf4j.Logger;
 
 import java.net.Inet4Address;
 import java.net.InetAddress;
 import java.net.URL;
 import java.net.UnknownHostException;
+import java.time.Duration;
+import java.time.Instant;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
@@ -50,12 +53,19 @@ import java.util.concurrent.Callable;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
+import java.util.function.BooleanSupplier;
 import java.util.function.Predicate;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 import java.util.stream.Stream;
 
 public class TestUtils {
+
+    public interface TimeoutHandler<X extends Throwable> {
+        void timeout() throws X;
+    }
+
     private static final Random R = new Random();
     private static Logger log = CustomLogger.getLogger();
 
@@ -368,34 +378,29 @@ public class TestUtils {
      * Check if endpoint is accessible
      */
     public static boolean resolvable(Endpoint endpoint) {
-        for (int i = 0; i < 10; i++) {
+        return waitUntilCondition(() -> {
             try {
                 InetAddress[] addresses = Inet4Address.getAllByName(endpoint.getHost());
                 return addresses.length > 0;
             } catch (UnknownHostException ignore) {
             }
-            try {
-                Thread.sleep(1_000);
-            } catch (InterruptedException e) {
-            }
-        }
-        return false;
+            return false;
+        }, Duration.ofSeconds(10), Duration.ofSeconds(1));
     }
 
     /**
      * Wait until Namespace will be removed
      *
      * @param kubernetes client for manipulation with kubernetes cluster
-     * @param namespace  project/namespace to remove
+     * @param namespace project/namespace to remove
      */
     public static void waitForNamespaceDeleted(Kubernetes kubernetes, String namespace) throws Exception {
-        TimeoutBudget budget = new TimeoutBudget(10, TimeUnit.MINUTES);
-        while (budget.timeLeft() >= 0 && kubernetes.listNamespaces().contains(namespace)) {
-            Thread.sleep(1000);
-        }
-        if (kubernetes.listNamespaces().contains(namespace)) {
-            throw new TimeoutException("Timed out waiting for namespace " + namespace + " to disappear");
-        }
+        waitUntilCondition(
+                () -> !kubernetes.listNamespaces().contains(namespace),
+                Duration.ofMinutes(10), Duration.ofSeconds(1),
+                () -> {
+                    throw new TimeoutException("Timed out waiting for namespace " + namespace + " to disappear");
+                });
     }
 
     /**
@@ -557,26 +562,131 @@ public class TestUtils {
     }
 
     public static void waitUntilCondition(final String forWhat, final Predicate<WaitPhase> condition, final TimeoutBudget budget) throws Exception {
+
         Objects.requireNonNull(condition);
         Objects.requireNonNull(budget);
 
         log.info("Waiting {} ms for - {}", budget.timeLeft(), forWhat);
 
-        while (!budget.timeoutExpired()) {
-            if (condition.test(WaitPhase.LOOP)) {
-                log.info("Successfully wait for: {} , it took {} ms", forWhat, budget.timeSpent());
-                return;
-            }
-            log.debug("next iteration, remaining time: {}", budget.timeLeft());
-            Thread.sleep(5000);
+        waitUntilCondition(
+
+                () -> condition.test(WaitPhase.LOOP),
+                budget.remaining(), Duration.ofSeconds(5),
+
+                () -> {
+                    // try once more
+                    if (condition.test(WaitPhase.LAST_TRY)) {
+                        log.info("Successfully wait for: {} , it passed on last try", forWhat);
+                        return;
+                    }
+
+                    throw new IllegalStateException("Failed to wait for: " + forWhat);
+                });
+
+        log.info("Successfully wait for: {}, it took {} ms", forWhat, budget.timeSpent());
+
+    }
+
+    /**
+     * Wait for a condition, fail otherwise.
+     *
+     * @param condition The condition to check, returning {@code true} means success.
+     * @param budget The remaining time budget for this step.
+     * @param delay The delay between checks.
+     * @param timeoutMessageSupplier The supplier of a timeout message.
+     * @throws AssertionFailedError In case the timeout expired
+     */
+    public static void waitUntilConditionOrFail(final BooleanSupplier condition, final Duration timeout, final Duration delay, final Supplier<String> timeoutMessageSupplier) {
+
+        Objects.requireNonNull(timeoutMessageSupplier);
+
+        waitUntilConditionOrThrow(condition, timeout, delay, () -> new AssertionFailedError(timeoutMessageSupplier.get()));
+
+    }
+
+    /**
+     * Wait for a condition, throw exception otherwise.
+     *
+     * @param condition The condition to check, returning {@code true} means success.
+     * @param budget The remaining time budget for this step.
+     * @param delay The delay between checks.
+     * @param exceptionSupplier The supplier of the exception to throw.
+     * @throws AssertionFailedError In case the timeout expired
+     */
+    public static <X extends Throwable> void waitUntilConditionOrThrow(final BooleanSupplier condition, final Duration timeout, final Duration delay, final Supplier<X> exceptionSupplier) throws X {
+
+        Objects.requireNonNull(exceptionSupplier);
+
+        waitUntilCondition(condition, timeout, delay, () -> {
+            throw exceptionSupplier.get();
+        });
+
+    }
+
+
+    /**
+     * Wait for a condition, call handler otherwise.
+     *
+     * @param condition The condition to check, returning {@code true} means success.
+     * @param budget The remaining time budget for this step.
+     * @param delay The delay between checks.
+     * @param timeoutHandler The handler to call in case of the timeout.
+     * @param <X> The type of exception thrown by the timeout handler.
+     * @throws AssertionFailedError In case the timeout expired
+     */
+    public static <X extends Throwable> void waitUntilCondition(final BooleanSupplier condition, final Duration timeout, final Duration delay, final TimeoutHandler<X> timeoutHandler) throws X {
+
+        Objects.requireNonNull(timeoutHandler);
+
+        if (!waitUntilCondition(condition, timeout, delay)) {
+            timeoutHandler.timeout();
         }
 
-        if (condition.test(WaitPhase.LAST_TRY)) {
-            log.info("Successfully wait for: {} , it passed on last try", forWhat);
-            return;
+    }
+
+    /**
+     * Wait for condition, return result.
+     * <p>
+     * This will check will put a priority on checking the condition, and only wait, when there is remaining time budget left.
+     * @param condition The condition to check, returning {@code true} means success.
+     * @param budget The remaining time budget for this step.
+     * @param delay The delay between checks.
+     * @return {@code true} if the condition was met, {@code false otherwise}.
+     */
+    public static boolean waitUntilCondition(final BooleanSupplier condition, final Duration timeout, final Duration delay) {
+
+        Objects.requireNonNull(condition);
+        Objects.requireNonNull(timeout);
+        Objects.requireNonNull(delay);
+
+        final Instant deadline = Instant.now().plus(timeout);
+
+        while ( true) {
+
+            // test first
+
+            if (condition.getAsBoolean()) {
+                return true;
+            }
+
+            // if the timeout has expired ... stop
+
+            final Duration remaining = Duration.between(deadline, Instant.now());
+            if ( !remaining.isNegative() ) {
+                return false;
+            }
+
+            // otherwise sleep
+
+            log.debug("next iteration, remaining time: {}", remaining);
+            try {
+                Thread.sleep(delay.toMillis());
+            } catch (InterruptedException e) {
+                throw new RuntimeException(e);
+            }
+
         }
 
-        throw new IllegalStateException("Failed to wait for: " + forWhat);
     }
 
     public static void waitForChangedResourceVersion(final TimeoutBudget budget, final String namespace, final String name, final String currentResourceVersion) throws Exception {
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/bases/IoTTestBase.java b/systemtests/src/test/java/io/enmasse/systemtest/bases/IoTTestBase.java
index 0b60def77..836e6c5da 100644
--- a/systemtests/src/test/java/io/enmasse/systemtest/bases/IoTTestBase.java
+++ b/systemtests/src/test/java/io/enmasse/systemtest/bases/IoTTestBase.java
@@ -40,6 +40,13 @@ public abstract class IoTTestBase extends TestBase implements ITestBaseIsolated
     protected static final String IOT_ADDRESS_CONTROL = "control";
     protected static final String IOT_ADDRESS_COMMAND = "command";
     protected static final String IOT_ADDRESS_COMMAND_RESPONSE = "command_response";
+    protected static final String[] IOT_ADDRESSES = new String[] {
+                    IOT_ADDRESS_TELEMETRY,
+                    IOT_ADDRESS_EVENT,
+                    IOT_ADDRESS_CONTROL,
+                    IOT_ADDRESS_COMMAND,
+                    IOT_ADDRESS_COMMAND_RESPONSE,
+    };
 
     private static Logger log = CustomLogger.getLogger();
     protected String iotProjectNamespace = "iot-project-ns";
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/iot/IoTProjectManagedTest.java b/systemtests/src/test/java/io/enmasse/systemtest/iot/IoTProjectManagedTest.java
index 2adab8bc5..d239bbd90 100644
--- a/systemtests/src/test/java/io/enmasse/systemtest/iot/IoTProjectManagedTest.java
+++ b/systemtests/src/test/java/io/enmasse/systemtest/iot/IoTProjectManagedTest.java
@@ -138,7 +138,7 @@ class IoTProjectManagedTest extends IoTTestBase implements ITestIsolatedStandard
 
         //username "adapter"
         //name "project-address-space"+".adapter"
-        User user = CommonResourcesManager.getInstance().getUser(addressSpace, "adapter");
+        User user = CommonResourcesManager.getInstance().getUser(addressSpace, "adapter-" + project.getMetadata().getUid());
         assertNotNull(user);
         assertEquals(1, user.getMetadata().getOwnerReferences().size());
         assertTrue(isOwner(project, user.getMetadata().getOwnerReferences().get(0)));
@@ -150,7 +150,7 @@ class IoTProjectManagedTest extends IoTTestBase implements ITestIsolatedStandard
         assertThat(authorizations, containsInAnyOrder(
                 asList(
                         assertAdapterAuthorization( of(send), expandAddresses(addressSuffix, IOT_ADDRESS_TELEMETRY, IOT_ADDRESS_EVENT, IOT_ADDRESS_COMMAND_RESPONSE)),
-                        assertAdapterAuthorization(of(recv), expandAddresses(addressSuffix, IOT_ADDRESS_COMMAND)),
+                        assertAdapterAuthorization( of(recv), expandAddresses(addressSuffix, IOT_ADDRESS_COMMAND)),
                         assertAdapterAuthorization( of(recv, send), expandAddresses(addressSuffix, IOT_ADDRESS_CONTROL)))));
     }
 
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/iot/project/AbstractIoTProjectTestBase.java b/systemtests/src/test/java/io/enmasse/systemtest/iot/project/AbstractIoTProjectTestBase.java
new file mode 100644
index 000000000..96c9744bd
--- /dev/null
+++ b/systemtests/src/test/java/io/enmasse/systemtest/iot/project/AbstractIoTProjectTestBase.java
@@ -0,0 +1,74 @@
+/*
+ * Copyright 2019, EnMasse authors.
+ * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html).
+ */
+
+package io.enmasse.systemtest.iot.project;
+
+
+
+import java.nio.ByteBuffer;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.extension.ExtensionContext;
+
+import io.enmasse.iot.model.v1.IoTConfig;
+import io.enmasse.iot.model.v1.IoTConfigBuilder;
+import io.enmasse.systemtest.Endpoint;
+import io.enmasse.systemtest.bases.DefaultDeviceRegistry;
+import io.enmasse.systemtest.bases.IoTTestBase;
+import io.enmasse.systemtest.bases.isolated.ITestIsolatedStandard;
+import io.enmasse.systemtest.certs.CertBundle;
+import io.enmasse.systemtest.iot.CredentialsRegistryClient;
+import io.enmasse.systemtest.iot.DeviceRegistryClient;
+import io.enmasse.systemtest.platform.apps.SystemtestsKubernetesApps;
+import io.enmasse.systemtest.utils.CertificateUtils;
+
+public abstract class AbstractIoTProjectTestBase extends IoTTestBase implements ITestIsolatedStandard {
+
+    protected DeviceRegistryClient registryClient;
+    protected CredentialsRegistryClient credentialsClient;
+
+    @BeforeEach
+    void initEnv() throws Exception {
+        CertBundle certBundle = CertificateUtils.createCertBundle();
+        IoTConfig iotConfig = new IoTConfigBuilder()
+                .withNewMetadata()
+                .withName("default")
+                .endMetadata()
+                .withNewSpec()
+                .withNewServices()
+
+                .withDeviceRegistry(DefaultDeviceRegistry.newInfinispanBased())
+
+                .endServices()
+                .withNewAdapters()
+                .withNewMqtt()
+                .withNewEndpoint()
+                .withNewKeyCertificateStrategy()
+                .withCertificate(ByteBuffer.wrap(certBundle.getCert().getBytes()))
+                .withKey(ByteBuffer.wrap(certBundle.getKey().getBytes()))
+                .endKeyCertificateStrategy()
+                .endEndpoint()
+                .endMqtt()
+                .endAdapters()
+                .endSpec()
+                .build();
+
+        createIoTConfig(iotConfig);
+
+        final Endpoint deviceRegistryEndpoint = kubernetes.getExternalEndpoint("device-registry");
+        registryClient = new DeviceRegistryClient(kubernetes, deviceRegistryEndpoint);
+        credentialsClient = new CredentialsRegistryClient(kubernetes, deviceRegistryEndpoint);
+    }
+
+    @AfterEach
+    void cleanEnv(ExtensionContext context) throws Exception {
+        if (context.getExecutionException().isPresent()) { //test failed
+            logCollector.collectHttpAdapterQdrProxyState();
+        }
+
+        SystemtestsKubernetesApps.deleteInfinispanServer(kubernetes.getInfraNamespace());
+    }
+
+}
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/iot/project/ManagedTest.java b/systemtests/src/test/java/io/enmasse/systemtest/iot/project/ManagedTest.java
new file mode 100644
index 000000000..583d3cc6f
--- /dev/null
+++ b/systemtests/src/test/java/io/enmasse/systemtest/iot/project/ManagedTest.java
@@ -0,0 +1,120 @@
+/*
+ * Copyright 2019, EnMasse authors.
+ * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html).
+ */
+package io.enmasse.systemtest.iot.project;
+
+import static io.enmasse.systemtest.utils.AddressSpaceUtils.addressSpaceExists;
+import static io.enmasse.systemtest.utils.TestUtils.waitUntilConditionOrFail;
+import static java.time.Duration.ofMinutes;
+import static java.time.Duration.ofSeconds;
+
+import java.util.function.BiConsumer;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+
+import io.enmasse.address.model.Address;
+import io.enmasse.address.model.AddressList;
+import io.enmasse.address.model.AddressSpace;
+import io.enmasse.address.model.AddressSpaceList;
+import io.enmasse.address.model.DoneableAddress;
+import io.enmasse.address.model.DoneableAddressSpace;
+import io.enmasse.address.model.KubeUtil;
+import io.enmasse.iot.model.v1.DoneableIoTProject;
+import io.enmasse.iot.model.v1.IoTProject;
+import io.enmasse.iot.model.v1.IoTProjectBuilder;
+import io.enmasse.iot.model.v1.IoTProjectList;
+import io.enmasse.systemtest.TestTag;
+import io.enmasse.systemtest.logs.CustomLogger;
+import io.enmasse.systemtest.utils.IoTUtils;
+import io.enmasse.user.model.v1.DoneableUser;
+import io.enmasse.user.model.v1.User;
+import io.enmasse.user.model.v1.UserList;
+import io.fabric8.kubernetes.client.dsl.MixedOperation;
+import io.fabric8.kubernetes.client.dsl.Resource;
+
+@Tag(TestTag.SHARED_IOT)
+public class ManagedTest extends AbstractIoTProjectTestBase  {
+
+    private static final Logger log = CustomLogger.getLogger();
+
+    private MixedOperation<IoTProject, IoTProjectList, DoneableIoTProject, Resource<IoTProject, DoneableIoTProject>> client;
+    private MixedOperation<Address, AddressList, DoneableAddress, Resource<Address, DoneableAddress>> addressClient;
+    private MixedOperation<AddressSpace, AddressSpaceList, DoneableAddressSpace, Resource<AddressSpace, DoneableAddressSpace>> addressSpaceClient;
+    private MixedOperation<User, UserList, DoneableUser, Resource<User, DoneableUser>> userClient;
+
+    @BeforeEach
+    public void initClients () {
+        this.client = kubernetes.getIoTProjectClient(this.iotProjectNamespace);
+        this.addressClient = kubernetes.getAddressClient(this.iotProjectNamespace);
+        this.addressSpaceClient = kubernetes.getAddressSpaceClient(this.iotProjectNamespace);
+        this.userClient = kubernetes.getUserClient(this.iotProjectNamespace);
+    }
+
+    @Test
+    public void testChangeAddressSpace() throws Exception {
+
+        var project = IoTUtils.getBasicIoTProjectObject("iot1", "as1", this.iotProjectNamespace);
+        createIoTProject(project);
+
+        assertManagedResources(Assertions::assertNotNull, project, "as1");
+
+        project = new IoTProjectBuilder(project)
+                .editSpec()
+                .editDownstreamStrategy()
+                .editManagedStrategy()
+                .editAddressSpace()
+
+                .withName("as1a")
+
+                .endAddressSpace()
+                .endManagedStrategy()
+                .endDownstreamStrategy()
+                .endSpec()
+                .build();
+
+        // update the project
+
+        log.info("Update project namespace");
+        client.createOrReplace(project);
+
+        // immediately after the change, the project is still ready but the new
+        // address space is still missing, so we need to wait for it to be created
+        // otherwise io.enmasse.systemtest.utils.IoTUtils.waitForIoTProjectReady(Kubernetes, IoTProject) will fail
+
+        waitUntilConditionOrFail(
+                addressSpaceExists(project.getMetadata().getNamespace(), project.getSpec().getDownstreamStrategy().getManagedStrategy().getAddressSpace().getName()),
+                ofMinutes(5), ofSeconds(10),
+                () -> String.format("Expected address space to be created"));
+
+        // wait until the project and address space become ready
+
+        log.info("For for project to become ready again");
+        IoTUtils.waitForIoTProjectReady(kubernetes, project);
+
+        // assert existence
+
+        assertManagedResources(Assertions::assertNotNull, project, "as1a");
+        assertManagedResources(Assertions::assertNull, project, "as1");
+
+    }
+
+    private void assertManagedResources(final BiConsumer<Object,String> assertor, final IoTProject project, final String addressSpaceName) {
+        assertObject(assertor, "Address space", this.addressSpaceClient, addressSpaceName);
+        assertObject(assertor, "Adapter user", this.userClient, addressSpaceName + ".adapter-" + project.getMetadata().getUid());
+        for ( final String address : IOT_ADDRESSES) {
+            var addressName = address + "/" + this.iotProjectNamespace + "." + project.getMetadata().getName();
+            var metaName = KubeUtil.sanitizeForGo(addressSpaceName, addressName);
+            assertObject(assertor, "Address: " + addressName + " / " + metaName, this.addressClient, metaName);
+        }
+    }
+
+    private static void assertObject(final BiConsumer<Object,String> assertor, final String message, final MixedOperation<? extends Object, ?, ?, ?> client, final String name) {
+        var object = client.withName(name).get();
+        assertor.accept(object, message);
+    }
+
+}
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/shared/standard/api/UserApiTest.java b/systemtests/src/test/java/io/enmasse/systemtest/shared/standard/api/UserApiTest.java
index fa79ef699..781fea166 100644
--- a/systemtests/src/test/java/io/enmasse/systemtest/shared/standard/api/UserApiTest.java
+++ b/systemtests/src/test/java/io/enmasse/systemtest/shared/standard/api/UserApiTest.java
@@ -15,12 +15,17 @@ import io.enmasse.systemtest.bases.shared.ITestSharedStandard;
 import io.enmasse.systemtest.cmdclients.KubeCMDClient;
 import io.enmasse.systemtest.logs.CustomLogger;
 import io.enmasse.systemtest.model.addressplan.DestinationPlan;
+import io.enmasse.systemtest.platform.Kubernetes;
 import io.enmasse.systemtest.utils.AddressUtils;
 import io.enmasse.systemtest.utils.UserUtils;
 import io.enmasse.user.model.v1.DoneableUser;
 import io.enmasse.user.model.v1.Operation;
 import io.enmasse.user.model.v1.User;
 import io.enmasse.user.model.v1.UserAuthorizationBuilder;
+import io.enmasse.user.model.v1.UserBuilder;
+import io.enmasse.user.model.v1.UserCrd;
+import io.enmasse.user.model.v1.UserList;
+
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
@@ -34,6 +39,8 @@ import java.util.concurrent.TimeUnit;
 
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.IsNull.nullValue;
+import static org.junit.Assert.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
@@ -134,4 +141,50 @@ public class UserApiTest extends TestBase implements ITestSharedStandard {
             kubernetes.deleteServiceAccount("test-service-account", environment.namespace());
         }
     }
+
+    @Test
+    void testSetAnnotations() {
+
+        var client = Kubernetes.getInstance().getClient();
+        var userCrd = client.customResources(UserCrd.messagingUser()   , User.class, UserList.class, DoneableUser.class);
+
+        var name = getSharedAddressSpace().getMetadata().getName()+ ".foo";
+        var user = new UserBuilder()
+                .withNewMetadata()
+                .withNamespace(getSharedAddressSpace().getMetadata().getNamespace())
+                .withName(name)
+                .addToAnnotations("annotation1", "value1")
+                .addToAnnotations("annotation2", "value2")
+                .endMetadata()
+                .build();
+
+        // create
+        userCrd.create(user);
+
+        // read again
+        var actualUser = userCrd.inNamespace(getSharedAddressSpace().getMetadata().getNamespace()).withName(name).get();
+
+        assertNotNull(actualUser);
+        assertThat(actualUser.getAnnotation("annotation1"), is("value1"));
+        assertThat(actualUser.getAnnotation("annotation2"), is("value2"));
+
+        var updateUser = new UserBuilder(actualUser)
+                .editOrNewMetadata()
+                .addToAnnotations("annotation1", "value1a")
+                .removeFromAnnotations("annotation2")
+                .addToAnnotations("annotation3", "value3")
+                .endMetadata()
+                .build();
+
+        userCrd.createOrReplace(updateUser);
+
+        // read again
+        actualUser = userCrd.inNamespace(getSharedAddressSpace().getMetadata().getNamespace()).withName(name).get();
+
+        assertNotNull(actualUser);
+        assertThat(actualUser.getAnnotation("annotation1"), is("value1a"));
+        assertThat(actualUser.getAnnotation("annotation2"), nullValue());
+        assertThat(actualUser.getAnnotation("annotation3"), is("value3"));
+
+    }
 }