Repository: EnMasseProject/enmasse
Pull request #3798 (2020-01-29 21:24:13)

Add support for specifying annotations in pod template

lulf: @enmasse-ci run tests profile=isolated
enmasse-ci: ### :x: Test Summary :x:
**TEST_PROFILE**: isolated
**TEST_CASE:** 
**TOTAL:** 98
**PASS:** 90
**FAIL:** 2
**SKIP:** 6
**BUILD_NUMBER:** 201
### :heavy_exclamation_mark: Test Failures :heavy_exclamation_mark:
- testReceiveThroughConnector1 in io.enmasse.systemtest.isolated.bridging.ConnectorsTest
- testAccessLogs in io.enmasse.systemtest.isolated.CommonTest

Patch:
diff --git a/.github/scripts/setup_docker.sh b/.github/scripts/setup_docker.sh
index 8e9bc2663..66fcf6305 100755
--- a/.github/scripts/setup_docker.sh
+++ b/.github/scripts/setup_docker.sh
@@ -1,3 +1,4 @@
 #!/bin/bash
 sudo apt install docker.io
+sudo systemctl unmask docker
 sudo systemctl start docker
\ No newline at end of file
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 303aa9536..9668639e1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -15,6 +15,8 @@
 *  #3657 Configure router pod anti-affinity by default
 *  Upgrade to Infinispan 9.4.16
 * #3729: Add relatedImages to operator manifest for disconnected install
+* #3824: Bump netty to 4.1.45.Final
+
 
 * #3673: Add support for pod disruption budgets
 
diff --git a/address-space-controller/src/main/resources/templates/brokered-space-infra.yaml b/address-space-controller/src/main/resources/templates/brokered-space-infra.yaml
index 5b5db402f..fd3c411c8 100644
--- a/address-space-controller/src/main/resources/templates/brokered-space-infra.yaml
+++ b/address-space-controller/src/main/resources/templates/brokered-space-infra.yaml
@@ -195,6 +195,7 @@ objects:
     labels:
       monitoring-key: enmasse-tenants
       app: enmasse
+      component: broker
       infraType: brokered
       infraUuid: ${INFRA_UUID}
     name: broker-${INFRA_UUID}
diff --git a/address-space-controller/src/main/resources/templates/standard-space-infra.yaml b/address-space-controller/src/main/resources/templates/standard-space-infra.yaml
index 3c137808a..abccfc47b 100644
--- a/address-space-controller/src/main/resources/templates/standard-space-infra.yaml
+++ b/address-space-controller/src/main/resources/templates/standard-space-infra.yaml
@@ -444,6 +444,7 @@ objects:
       app: enmasse
       infraType: standard
       infraUuid: ${INFRA_UUID}
+      component: router
     name: qdrouterd-headless-${INFRA_UUID}
   spec:
     clusterIP: None
@@ -483,6 +484,7 @@ objects:
       addressSpace: ${ADDRESS_SPACE}
     labels:
       app: enmasse
+      component: admin
       infraType: standard
       infraUuid: ${INFRA_UUID}
     name: standard-controller-${INFRA_UUID}
diff --git a/api-model/src/main/java/io/enmasse/address/model/KubeUtil.java b/api-model/src/main/java/io/enmasse/address/model/KubeUtil.java
index 15df1786b..74ef9028a 100644
--- a/api-model/src/main/java/io/enmasse/address/model/KubeUtil.java
+++ b/api-model/src/main/java/io/enmasse/address/model/KubeUtil.java
@@ -4,16 +4,6 @@
  */
 package io.enmasse.address.model;
 
-import com.google.common.hash.Hashing;
-import io.enmasse.config.AnnotationKeys;
-import io.fabric8.kubernetes.api.model.Container;
-import io.fabric8.kubernetes.api.model.EnvVar;
-import io.fabric8.kubernetes.api.model.HasMetadata;
-import io.fabric8.kubernetes.api.model.PodSecurityContext;
-import io.fabric8.kubernetes.api.model.PodSpec;
-import io.fabric8.kubernetes.api.model.PodTemplateSpec;
-import io.fabric8.kubernetes.api.model.Probe;
-
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -25,6 +15,17 @@ import java.util.UUID;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
+import com.google.common.hash.Hashing;
+
+import io.enmasse.config.AnnotationKeys;
+import io.fabric8.kubernetes.api.model.Container;
+import io.fabric8.kubernetes.api.model.EnvVar;
+import io.fabric8.kubernetes.api.model.HasMetadata;
+import io.fabric8.kubernetes.api.model.PodSecurityContext;
+import io.fabric8.kubernetes.api.model.PodSpec;
+import io.fabric8.kubernetes.api.model.PodTemplateSpec;
+import io.fabric8.kubernetes.api.model.Probe;
+
 /**
  * Various static utilities that don't belong in a specific place
  */
@@ -101,15 +102,17 @@ public final class KubeUtil {
      */
     public static String sanitizeForGo(final String addressSpace, final String address) {
 
-        byte [] addressBytes = address.getBytes(StandardCharsets.UTF_8);
-        byte [] data = new byte[addressBytes.length + GO_NAMESPACE.length];
+        final byte [] addressBytes = address.getBytes(StandardCharsets.UTF_8);
+        final byte [] data = new byte[addressBytes.length + GO_NAMESPACE.length];
 
         System.arraycopy(GO_NAMESPACE, 0, data, 0, GO_NAMESPACE.length);
         System.arraycopy(addressBytes, 0, data, GO_NAMESPACE.length, addressBytes.length);
 
-        String uuid = UUID.nameUUIDFromBytes(data).toString();
+        final String uuid = UUID.nameUUIDFromBytes(data).toString();
+        final String part1 = KubeUtil.sanitizeName(addressSpace);
+        final String part2 = KubeUtil.sanitizeWithUuid(address, uuid, addressPattern);
 
-        return KubeUtil.sanitizeName(addressSpace) + "." + KubeUtil.sanitizeWithUuid(address, uuid, addressPattern);
+        return (part1 + "." + part2).replaceAll("-+", "-");
     }
 
     public static String getAddressSpaceCaSecretName(AddressSpace addressSpace) {
@@ -175,6 +178,14 @@ public final class KubeUtil {
             actual.getMetadata().setLabels(labels);
         }
 
+        if (desired.getMetadata() != null && desired.getMetadata().getAnnotations() != null) {
+            Map<String, String> annotations = new HashMap<>(desired.getMetadata().getAnnotations());
+            if (actual.getMetadata().getAnnotations() != null) {
+                annotations.putAll(actual.getMetadata().getAnnotations());
+            }
+            actual.getMetadata().setAnnotations(annotations);
+        }
+
         if (desired.getSpec() != null) {
 
             PodSpec podSpec = desired.getSpec();
diff --git a/controller-manager/src/main/resources/templates/ServiceMonitor-address-space-controller.yaml b/controller-manager/src/main/resources/templates/ServiceMonitor-address-space-controller.yaml
new file mode 100644
index 000000000..6668c622b
--- /dev/null
+++ b/controller-manager/src/main/resources/templates/ServiceMonitor-address-space-controller.yaml
@@ -0,0 +1,23 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: enmasse-address-space-controller
+  namespace: {{ index .Params "Namespace" }}
+  labels:
+    monitoring-key: middleware
+    app: enmasse
+spec:
+  selector:
+    matchLabels:
+      app: enmasse
+      component: address-space-controller
+  endpoints:
+  - port: health
+    path: /metrics
+    metricRelabelings:
+      - sourceLabels: [__name__]
+        targetLabel: __name__
+        replacement: "enmasse_${1}"
+  namespaceSelector:
+    matchLabels:
+      monitoring-key: middleware
diff --git a/controller-manager/src/main/resources/templates/ServiceMonitor-admin.yaml b/controller-manager/src/main/resources/templates/ServiceMonitor-admin.yaml
new file mode 100644
index 000000000..724b1af5d
--- /dev/null
+++ b/controller-manager/src/main/resources/templates/ServiceMonitor-admin.yaml
@@ -0,0 +1,22 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: enmasse-admin
+  namespace: {{ index .Params "Namespace" }}
+  labels:
+    monitoring-key: middleware
+    app: enmasse
+spec:
+  selector:
+    matchLabels:
+      app: enmasse
+      component: admin
+  endpoints:
+  - port: health
+    metricRelabelings:
+      - sourceLabels: [__name__]
+        targetLabel: __name__
+        replacement: "enmasse_${1}"
+  namespaceSelector:
+    matchLabels:
+      monitoring-key: middleware
diff --git a/controller-manager/src/main/resources/templates/ServiceMonitor-enmasse.yaml b/controller-manager/src/main/resources/templates/ServiceMonitor-broker.yaml
similarity index 69%
rename from controller-manager/src/main/resources/templates/ServiceMonitor-enmasse.yaml
rename to controller-manager/src/main/resources/templates/ServiceMonitor-broker.yaml
index 02ea1ec0e..5273b86f7 100644
--- a/controller-manager/src/main/resources/templates/ServiceMonitor-enmasse.yaml
+++ b/controller-manager/src/main/resources/templates/ServiceMonitor-broker.yaml
@@ -1,7 +1,7 @@
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
-  name: enmasse
+  name: enmasse-broker
   namespace: {{ index .Params "Namespace" }}
   labels:
     monitoring-key: middleware
@@ -10,14 +10,16 @@ spec:
   selector:
     matchLabels:
       app: enmasse
+      component: broker
   endpoints:
-  - port: health
   - port: health-tls
     scheme: https
     tlsConfig:
       insecureSkipVerify: true
-  - port: prometheus
-    path: /prometheus
+    metricRelabelings:
+      - sourceLabels: [__name__]
+        targetLabel: __name__
+        replacement: "enmasse_${1}"
   namespaceSelector:
     matchLabels:
       monitoring-key: middleware
diff --git a/controller-manager/src/main/resources/templates/ServiceMonitor-iot.yaml b/controller-manager/src/main/resources/templates/ServiceMonitor-iot.yaml
new file mode 100644
index 000000000..d427e37dd
--- /dev/null
+++ b/controller-manager/src/main/resources/templates/ServiceMonitor-iot.yaml
@@ -0,0 +1,23 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: enmasse-iot
+  namespace: {{ index .Params "Namespace" }}
+  labels:
+    monitoring-key: middleware
+    app: enmasse
+spec:
+  selector:
+    matchLabels:
+      app: enmasse
+      component: iot
+  endpoints:
+  - port: prometheus
+    path: /prometheus
+    metricRelabelings:
+      - sourceLabels: [__name__]
+        targetLabel: __name__
+        replacement: "enmasse_${1}"
+  namespaceSelector:
+    matchLabels:
+      monitoring-key: middleware
diff --git a/controller-manager/src/main/resources/templates/ServiceMonitor-router.yaml b/controller-manager/src/main/resources/templates/ServiceMonitor-router.yaml
new file mode 100644
index 000000000..58e4d893d
--- /dev/null
+++ b/controller-manager/src/main/resources/templates/ServiceMonitor-router.yaml
@@ -0,0 +1,22 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: enmasse-router
+  namespace: {{ index .Params "Namespace" }}
+  labels:
+    monitoring-key: middleware
+    app: enmasse
+spec:
+  selector:
+    matchLabels:
+      app: enmasse
+      component: router
+  endpoints:
+  - port: health
+    metricRelabelings:
+      - sourceLabels: [__name__]
+        targetLabel: __name__
+        replacement: "enmasse_${1}"
+  namespaceSelector:
+    matchLabels:
+      monitoring-key: middleware
diff --git a/documentation/modules/ref-metrics-common.adoc b/documentation/modules/ref-metrics-common.adoc
index 555ac31bf..c03b01ff6 100644
--- a/documentation/modules/ref-metrics-common.adoc
+++ b/documentation/modules/ref-metrics-common.adoc
@@ -7,8 +7,8 @@
 
 The following components export these common metrics:
 
+* `enmasse-operator`
 * `address-space-controller`
-* `api-server`
 * `standard-controller`
 
 
@@ -19,7 +19,7 @@ Example:::
 [source,options="nowrap"]
 ----
 enmasse_version{job="address-space-controller",version="1.0.1"} 1
-enmasse_version{job="api-server",version="1.0.1"} 1
+enmasse_version{job="enmsse-operator",version="1.0.1"} 1
 enmasse_version{job="standard-controller",version="1.0.1"} 1
 ----
 
diff --git a/hack/run-codegen.sh b/hack/run-codegen.sh
new file mode 100755
index 000000000..3d1adf8d2
--- /dev/null
+++ b/hack/run-codegen.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+#
+# Copyright 2018, EnMasse authors.
+# License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html).
+#
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+SCRIPTPATH="$(cd "$(dirname "$0")" && pwd -P)"
+
+"$SCRIPTPATH/generate-groups.sh" "all" \
+    github.com/enmasseproject/enmasse/pkg/client \
+    github.com/enmasseproject/enmasse/pkg/apis \
+    "admin:v1beta1 admin:v1beta2 enmasse:v1beta1 user:v1beta1 iot:v1alpha1" \
+    --go-header-file "${SCRIPTPATH}/header.txt" \
+    "$@"
diff --git a/hack/update-codegen.sh b/hack/update-codegen.sh
index 3d1adf8d2..32b99dbd2 100755
--- a/hack/update-codegen.sh
+++ b/hack/update-codegen.sh
@@ -11,9 +11,20 @@ set -o pipefail
 
 SCRIPTPATH="$(cd "$(dirname "$0")" && pwd -P)"
 
-"$SCRIPTPATH/generate-groups.sh" "all" \
-    github.com/enmasseproject/enmasse/pkg/client \
-    github.com/enmasseproject/enmasse/pkg/apis \
-    "admin:v1beta1 admin:v1beta2 enmasse:v1beta1 user:v1beta1 iot:v1alpha1" \
-    --go-header-file "${SCRIPTPATH}/header.txt" \
-    "$@"
+TMPBASE="$(mktemp -d)"
+TMPPROJ="$TMPBASE/github.com/enmasseproject/enmasse"
+
+cleanup() {
+    echo "Cleaning up: $TMPBASE"
+    rm -rf "$TMPBASE"
+}
+
+echo "Using tmp base: $TMPBASE"
+
+trap "cleanup" EXIT SIGINT
+
+mkdir -p "$TMPPROJ"
+
+"$SCRIPTPATH/run-codegen.sh" --output-base "$TMPBASE"
+
+cp -av "$TMPPROJ"/. .
diff --git a/hack/verify-codegen.sh b/hack/verify-codegen.sh
index 22912b7b4..c0f8b464f 100755
--- a/hack/verify-codegen.sh
+++ b/hack/verify-codegen.sh
@@ -26,7 +26,7 @@ trap "cleanup" EXIT SIGINT
 mkdir -p "$TMPPROJ"
 cp -dR "$SCRIPTPATH/../" "$TMPPROJ"
 
-"$SCRIPTPATH/update-codegen.sh" --output-base "$TMPBASE"
+"$SCRIPTPATH/run-codegen.sh" --output-base "$TMPBASE"
 
 echo "Comparing existing generated code with temporarily generated code"
 if diff -Nur --no-dereference "$SCRIPTPATH/.." "$TMPPROJ" -x "go-bin"; then
diff --git a/iot/iot-device-registry-base/src/main/java/io/enmasse/iot/registry/tenant/TenantInformation.java b/iot/iot-device-registry-base/src/main/java/io/enmasse/iot/registry/tenant/TenantInformation.java
index 3ab957e1e..de77c082d 100644
--- a/iot/iot-device-registry-base/src/main/java/io/enmasse/iot/registry/tenant/TenantInformation.java
+++ b/iot/iot-device-registry-base/src/main/java/io/enmasse/iot/registry/tenant/TenantInformation.java
@@ -16,8 +16,13 @@ public class TenantInformation extends TenantHandle {
 
     private final Optional<Tenant> tenant;
 
-    protected TenantInformation(final String name, final String id, final Tenant tenant) {
-        super(name,id);
+    private final String namespace;
+    private final String projectName;
+
+    protected TenantInformation(final String namespace, final String projectName, final String id, final Tenant tenant) {
+        super(namespace + "." + projectName, id);
+        this.namespace = namespace;
+        this.projectName = projectName;
         this.tenant = Optional.ofNullable(tenant);
     }
 
@@ -25,16 +30,25 @@ public class TenantInformation extends TenantHandle {
         return this.tenant;
     }
 
+    public String getNamespace() {
+        return namespace;
+    }
+
+    public String getProjectName() {
+        return projectName;
+    }
+
     @Override
     protected ToStringHelper toStringHelper() {
         return super.toStringHelper()
                 .add("tenant", this.tenant);
     }
 
-    public static TenantInformation of(final String name, final String id, final Tenant tenant) {
-        Objects.requireNonNull(name);
+    public static TenantInformation of(final String namespace, final String projectName, final String id, final Tenant tenant) {
+        Objects.requireNonNull(namespace);
+        Objects.requireNonNull(projectName);
         Objects.requireNonNull(id);
-        return new TenantInformation(name, id, tenant);
+        return new TenantInformation(namespace, projectName, id, tenant);
     }
 
 }
diff --git a/iot/iot-device-registry-base/src/main/java/io/enmasse/iot/registry/util/DeviceRegistryTokenAuthProvider.java b/iot/iot-device-registry-base/src/main/java/io/enmasse/iot/registry/util/DeviceRegistryTokenAuthProvider.java
index 65e857ad6..d9a586529 100644
--- a/iot/iot-device-registry-base/src/main/java/io/enmasse/iot/registry/util/DeviceRegistryTokenAuthProvider.java
+++ b/iot/iot-device-registry-base/src/main/java/io/enmasse/iot/registry/util/DeviceRegistryTokenAuthProvider.java
@@ -9,8 +9,15 @@ import io.enmasse.api.auth.KubeAuthApi;
 import io.enmasse.api.auth.RbacSecurityContext;
 import io.enmasse.api.auth.ResourceVerb;
 import io.enmasse.api.auth.TokenReview;
+import io.enmasse.iot.registry.tenant.TenantInformation;
+import io.enmasse.iot.registry.tenant.TenantInformationService;
+import static io.enmasse.iot.registry.util.DeviceRegistryTokenAuthHandler.METHOD;
+import static io.enmasse.iot.registry.util.DeviceRegistryTokenAuthHandler.TENANT;
+import static io.enmasse.iot.registry.util.DeviceRegistryTokenAuthHandler.TOKEN;
+
 import io.fabric8.kubernetes.client.DefaultKubernetesClient;
 import io.fabric8.kubernetes.client.NamespacedKubernetesClient;
+import io.opentracing.noop.NoopSpan;
 import io.vertx.core.AsyncResult;
 import io.vertx.core.Future;
 import io.vertx.core.Handler;
@@ -19,6 +26,7 @@ import io.vertx.core.json.JsonObject;
 import io.vertx.ext.auth.AuthProvider;
 import io.vertx.ext.auth.User;
 import io.vertx.ext.web.handler.impl.HttpStatusException;
+import static java.net.HttpURLConnection.HTTP_NOT_FOUND;
 
 import org.infinispan.Cache;
 import org.infinispan.configuration.cache.ConfigurationBuilder;
@@ -26,11 +34,7 @@ import org.infinispan.manager.DefaultCacheManager;
 import org.infinispan.manager.EmbeddedCacheManager;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-
-import static io.enmasse.iot.registry.util.DeviceRegistryTokenAuthHandler.METHOD;
-import static io.enmasse.iot.registry.util.DeviceRegistryTokenAuthHandler.TENANT;
-import static io.enmasse.iot.registry.util.DeviceRegistryTokenAuthHandler.TOKEN;
-
+import org.springframework.beans.factory.annotation.Autowired;
 import java.time.Duration;
 import java.util.concurrent.TimeUnit;
 
@@ -40,13 +44,15 @@ public class DeviceRegistryTokenAuthProvider implements AuthProvider {
 
     private static final HttpStatusException UNAUTHORIZED = new HttpStatusException(401);
 
+    protected TenantInformationService tenantInformationService;
+
     private final NamespacedKubernetesClient client;
     private final AuthApi authApi;
     private final EmbeddedCacheManager cacheManager = new DefaultCacheManager();
     private final Cache<String, TokenReview> tokens;
     private final Cache<String, Boolean> authorizations;
 
-    public DeviceRegistryTokenAuthProvider(Duration tokenExpiration) {
+    public DeviceRegistryTokenAuthProvider(final Duration tokenExpiration) {
         log.info("Using token cache expiration of {}", tokenExpiration);
         this.client = new DefaultKubernetesClient();
         this.authApi = new KubeAuthApi(this.client, this.client.getConfiguration().getOauthToken());
@@ -59,6 +65,11 @@ public class DeviceRegistryTokenAuthProvider implements AuthProvider {
 
     }
 
+    @Autowired
+    public void setTenantInformationService(final TenantInformationService tenantInformationService) {
+        this.tenantInformationService = tenantInformationService;
+    }
+
     @Override
     public void authenticate(final JsonObject authInfo, final Handler<AsyncResult<User>> resultHandler) {
 
@@ -70,27 +81,14 @@ public class DeviceRegistryTokenAuthProvider implements AuthProvider {
                 })
                 .thenAccept(tokenReview -> {
                     if (tokenReview != null && tokenReview.isAuthenticated()) {
-                        String role = getRole(authInfo);
-                        if (role != null) {
-                            final RbacSecurityContext securityContext = new RbacSecurityContext(tokenReview, authApi, null);
-                            authorizations
-                                    .computeIfAbsentAsync(role, authorized -> securityContext.isUserInRole(role))
-                                    .exceptionally(t-> {
-                                        log.info("Error performing authorization", t);
-                                        return false;
-                                    })
-                                    .thenAccept(authResult -> {
-                                        if (authResult) {
-                                            resultHandler.handle(Future.succeededFuture());
-                                        } else {
-                                            log.debug("Bearer token not authorized");
-                                            resultHandler.handle(Future.failedFuture(UNAUTHORIZED));
-                                        }
-                                    });
-                        } else {
-                            log.debug("Error parsing the role from {}", authInfo);
-                            resultHandler.handle(Future.failedFuture(UNAUTHORIZED));
-                        }
+                        tenantInformationService.tenantExists(authInfo.getString(TENANT), HTTP_NOT_FOUND, NoopSpan.INSTANCE)
+                                .exceptionally(e -> {
+                                    log.info("Tenant doesn't exists", e);
+                                    resultHandler.handle(Future.failedFuture(UNAUTHORIZED));
+                                    return null;
+                                })
+                                .thenAccept(tenant -> authorize(authInfo, tokenReview, tenant, resultHandler)
+                                );
                     } else {
                         log.debug("Bearer token not authenticated");
                         resultHandler.handle(Future.failedFuture(UNAUTHORIZED));
@@ -98,22 +96,27 @@ public class DeviceRegistryTokenAuthProvider implements AuthProvider {
                 });
     }
 
-    private String getRole(final JsonObject authInfo) {
+    private void authorize(final JsonObject authInfo, final TokenReview tokenReview, final TenantInformation tenant, final Handler<AsyncResult<User>> resultHandler) {
         final HttpMethod method = HttpMethod.valueOf(authInfo.getString(METHOD));
         ResourceVerb verb = ResourceVerb.update;
         if (method == HttpMethod.GET) {
             verb = ResourceVerb.get;
         }
-
-        final String[] tenant = authInfo.getString(TENANT).split("\\.");
-        if (tenant.length != 2) {
-            log.info("Wrong tenant format (namespace.project) for value '{}'", new Object[] {tenant});
-            return null;
-        }
-        final String namespace = tenant[0];
-        final String iotProject = tenant[1];
-
-        return RbacSecurityContext.rbacToRole(namespace, verb, "iotprojects", iotProject, "iot.enmasse.io");
+        String role = RbacSecurityContext.rbacToRole(tenant.getNamespace(), verb, "iotprojects", tenant.getProjectName(), "iot.enmasse.io");
+        RbacSecurityContext securityContext = new RbacSecurityContext(tokenReview, authApi, null);
+        authorizations
+                .computeIfAbsentAsync(role, authorized -> securityContext.isUserInRole(role))
+                .exceptionally(t -> {
+                    log.info("Error performing authorization", t);
+                    return false;
+                })
+                .thenAccept(authResult -> {
+                    if (authResult) {
+                        resultHandler.handle(Future.succeededFuture());
+                    } else {
+                        log.debug("Bearer token not authorized");
+                        resultHandler.handle(Future.failedFuture(UNAUTHORIZED));
+                    }
+                });
     }
-
 }
\ No newline at end of file
diff --git a/iot/iot-device-registry-infinispan/src/main/java/io/enmasse/iot/registry/infinispan/config/DeviceServiceConfiguration.java b/iot/iot-device-registry-infinispan/src/main/java/io/enmasse/iot/registry/infinispan/config/DeviceServiceConfiguration.java
index 6acc419d0..302b4a09a 100644
--- a/iot/iot-device-registry-infinispan/src/main/java/io/enmasse/iot/registry/infinispan/config/DeviceServiceConfiguration.java
+++ b/iot/iot-device-registry-infinispan/src/main/java/io/enmasse/iot/registry/infinispan/config/DeviceServiceConfiguration.java
@@ -21,6 +21,7 @@ import org.springframework.context.annotation.Configuration;
 import io.enmasse.iot.registry.util.DeviceRegistryTokenAuthHandler;
 import io.enmasse.iot.registry.util.DeviceRegistryTokenAuthProvider;
 import io.vertx.core.Vertx;
+import io.vertx.ext.auth.AuthProvider;
 import io.vertx.ext.web.handler.AuthHandler;
 
 @Configuration
@@ -80,9 +81,13 @@ public class DeviceServiceConfiguration {
     @Bean
     @Autowired
     public AuthHandler authHandler(final RestEndpointConfiguration restEndpointConfiguration) {
-        return new DeviceRegistryTokenAuthHandler(
-                new DeviceRegistryTokenAuthProvider(restEndpointConfiguration.getAuthTokenCacheExpiration()
-                ));
+        return new DeviceRegistryTokenAuthHandler(authProvider(restEndpointConfiguration));
+    }
+
+    @Bean
+    @Autowired
+    public AuthProvider authProvider(final RestEndpointConfiguration restEndpointConfiguration) {
+        return new DeviceRegistryTokenAuthProvider(restEndpointConfiguration.getAuthTokenCacheExpiration());
     }
 
     /**
diff --git a/iot/pom.xml b/iot/pom.xml
index a534b3cfb..1e3baac16 100644
--- a/iot/pom.xml
+++ b/iot/pom.xml
@@ -13,7 +13,7 @@
   <packaging>pom</packaging>
 
   <properties>
-    <hono.version>1.0.2</hono.version>
+    <hono.version>1.0.3</hono.version>
     <infinispan.version>9.4.16.Final</infinispan.version>
 
     <!-- align with Hono compatible versions -->
diff --git a/metrics-api/src/main/java/io/enmasse/metrics/api/PrometheusMetricsFormatter.java b/metrics-api/src/main/java/io/enmasse/metrics/api/PrometheusMetricsFormatter.java
index ea020eff0..a516dda91 100644
--- a/metrics-api/src/main/java/io/enmasse/metrics/api/PrometheusMetricsFormatter.java
+++ b/metrics-api/src/main/java/io/enmasse/metrics/api/PrometheusMetricsFormatter.java
@@ -13,8 +13,8 @@ public class PrometheusMetricsFormatter implements MetricsFormatter {
 
         for (Metric metric : metricList) {
             if (!MetricType.up.equals(metric.getType())) {
-                sb.append("# HELP ").append("enmasse_").append(metric.getName()).append(" ").append(metric.getDescription()).append("\n");
-                sb.append("# TYPE ").append("enmasse_").append(metric.getName()).append(" ").append(metric.getType().name()).append("\n");
+                sb.append("# HELP ").append(metric.getName()).append(" ").append(metric.getDescription()).append("\n");
+                sb.append("# TYPE ").append(metric.getName()).append(" ").append(metric.getType().name()).append("\n");
             }
             String suffix = metric.getType().equals(MetricType.histogram) ? "_bucket" : "";
             MetricSnapshot snapshot = metric.getSnapshot();
@@ -35,7 +35,7 @@ public class PrometheusMetricsFormatter implements MetricsFormatter {
 
     private static String formatMetricValue(String name, MetricValue metricValue, long timestamp) {
         StringBuilder sb = new StringBuilder();
-        sb.append("enmasse_").append(name);
+        sb.append(name);
         if (!metricValue.getLabels().isEmpty()) {
             sb.append("{");
             sb.append(metricValue.getLabels().stream()
diff --git a/pkg/controller/iotproject/iotproject_controller.go b/pkg/controller/iotproject/iotproject_controller.go
index ad28c1180..b6ab3c67e 100644
--- a/pkg/controller/iotproject/iotproject_controller.go
+++ b/pkg/controller/iotproject/iotproject_controller.go
@@ -86,8 +86,7 @@ func add(mgr manager.Manager, r *ReconcileIoTProject) error {
 	// Watch for address spaces
 
 	err = c.Watch(&source.Kind{Type: &enmassev1beta1.AddressSpace{}}, &handler.EnqueueRequestForOwner{
-		IsController: true,
-		OwnerType:    &iotv1alpha1.IoTProject{},
+		OwnerType: &iotv1alpha1.IoTProject{},
 	})
 	if err != nil {
 		return err
diff --git a/pkg/controller/iotproject/managed.go b/pkg/controller/iotproject/managed.go
index b5f3458fd..5e72a28cb 100644
--- a/pkg/controller/iotproject/managed.go
+++ b/pkg/controller/iotproject/managed.go
@@ -312,6 +312,15 @@ func (r *ReconcileIoTProject) reconcileAddressSpace(ctx context.Context, project
 
 	rc, err := controllerutil.CreateOrUpdate(ctx, r.client, addressSpace, func() error {
 
+		if addressSpace.DeletionTimestamp != nil {
+			// address space is deconstructed
+			managedStatus.remainingReady[resourceTypeAddressSpace] = false
+			managedStatus.remainingCreated[resourceTypeAddressSpace] = false
+			// re-try later to re-create
+			retryDelay = 10 * time.Second
+			return nil
+		}
+
 		managedStatus.remainingReady[resourceTypeAddressSpace] = addressSpace.Status.IsReady
 
 		// if the address space is not ready yet
diff --git a/pkg/util/templatehelper.go b/pkg/util/templatehelper.go
index ec2334d8e..37726eca1 100644
--- a/pkg/util/templatehelper.go
+++ b/pkg/util/templatehelper.go
@@ -52,7 +52,11 @@ func GetTemplateList() []string {
 	templateList := []string{
 		"PrometheusRules-kube-metrics.yaml",
 		"PrometheusRules-enmasse.yaml",
-		"ServiceMonitor-enmasse.yaml",
+		"ServiceMonitor-address-space-controller.yaml",
+		"ServiceMonitor-admin.yaml",
+		"ServiceMonitor-router.yaml",
+		"ServiceMonitor-broker.yaml",
+		"ServiceMonitor-iot.yaml",
 		"GrafanaDashboard-brokers.yaml",
 		"GrafanaDashboard-components.yaml",
 		"GrafanaDashboard-routers.yaml",
diff --git a/pom.xml b/pom.xml
index 59fde006a..afec02ca4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -70,7 +70,7 @@
         <artemis.version>2.10.1</artemis.version>
         <ngwebdriver.version>1.1.4</ngwebdriver.version>
         <paho.version>1.2.1</paho.version>
-        <netty.version>4.1.34.Final</netty.version>
+        <netty.version>4.1.45.Final</netty.version>
         <keycloak.version>4.8.3.Final</keycloak.version>
         <jboss.logging.version>3.3.0.Final</jboss.logging.version>
         <jboss.logging.processor.version>2.1.0.Final</jboss.logging.processor.version>
@@ -92,7 +92,7 @@
         <frontend-maven-plugin.version>1.6</frontend-maven-plugin.version>
         <properties-maven-plugin.version>1.0.0</properties-maven-plugin.version>
         <maven-resources-plugin.version>3.1.0</maven-resources-plugin.version>
-        <maven-checkstyle-plugin.version>3.0.0</maven-checkstyle-plugin.version>
+        <maven-checkstyle-plugin.version>3.1.0</maven-checkstyle-plugin.version>
         <maven-compiler-plugin.version>3.8.0</maven-compiler-plugin.version>
         <maven-jar-plugin.version>3.1.1</maven-jar-plugin.version>
         <maven-shade-plugin.version>3.1.0</maven-shade-plugin.version>
@@ -103,7 +103,7 @@
         <security-versions-plugin.version>1.0.6</security-versions-plugin.version>
         <dependency-check-maven-plugin.version>4.0.2</dependency-check-maven-plugin.version>
         <maven-xml-plugin.version>1.0.2</maven-xml-plugin.version>
-
+        <checkstyle.version>8.29</checkstyle.version>
         <license-report-stylesheet>${maven.multiModuleProjectDirectory}/licenses.xsl</license-report-stylesheet>
     </properties>
 
@@ -821,7 +821,7 @@
                     <dependency>
                         <groupId>com.puppycrawl.tools</groupId>
                         <artifactId>checkstyle</artifactId>
-                        <version>8.18</version>
+                        <version>${checkstyle.version}</version>
                     </dependency>
                 </dependencies>
                 <executions>
diff --git a/standard-controller/src/main/resources/templates/queue-persisted.yaml b/standard-controller/src/main/resources/templates/queue-persisted.yaml
index 075b48f5e..47c042520 100644
--- a/standard-controller/src/main/resources/templates/queue-persisted.yaml
+++ b/standard-controller/src/main/resources/templates/queue-persisted.yaml
@@ -14,6 +14,7 @@ objects:
     labels:
       monitoring-key: enmasse-tenants
       app: enmasse
+      component: broker
       infraType: standard
       infraUuid: ${INFRA_UUID}
     name: ${NAME}
diff --git a/standard-controller/src/main/resources/templates/topic-persisted.yaml b/standard-controller/src/main/resources/templates/topic-persisted.yaml
index 381046cd9..838f1cfaf 100644
--- a/standard-controller/src/main/resources/templates/topic-persisted.yaml
+++ b/standard-controller/src/main/resources/templates/topic-persisted.yaml
@@ -14,6 +14,7 @@ objects:
     labels:
       monitoring-key: enmasse-tenants
       app: enmasse
+      component: broker
       infraType: standard
       infraUuid: ${INFRA_UUID}
     name: ${NAME}


Test patch:
diff --git a/api-model/src/test/java/io/enmasse/address/model/v1/address/KubeUtilTest.java b/api-model/src/test/java/io/enmasse/address/model/v1/address/KubeUtilTest.java
index c55c51d3b..9462d0038 100644
--- a/api-model/src/test/java/io/enmasse/address/model/v1/address/KubeUtilTest.java
+++ b/api-model/src/test/java/io/enmasse/address/model/v1/address/KubeUtilTest.java
@@ -4,9 +4,14 @@
  */
 package io.enmasse.address.model.v1.address;
 
-import io.enmasse.address.model.KubeUtil;
-
-import static org.hamcrest.Matchers.*;
+import static java.util.stream.Stream.concat;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
+import static org.hamcrest.Matchers.nullValue;
+import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.params.provider.Arguments.of;
 
@@ -15,18 +20,24 @@ import java.util.List;
 import java.util.Map;
 import java.util.stream.Stream;
 
-import io.fabric8.kubernetes.api.model.*;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.Arguments;
 import org.junit.jupiter.params.provider.MethodSource;
 
-import static java.util.stream.Stream.concat;
-import static org.hamcrest.CoreMatchers.is;
-import static org.hamcrest.MatcherAssert.assertThat;
+import io.enmasse.address.model.KubeUtil;
+import io.fabric8.kubernetes.api.model.Container;
+import io.fabric8.kubernetes.api.model.ContainerBuilder;
+import io.fabric8.kubernetes.api.model.EnvVar;
+import io.fabric8.kubernetes.api.model.EnvVarBuilder;
+import io.fabric8.kubernetes.api.model.PodTemplateSpec;
+import io.fabric8.kubernetes.api.model.PodTemplateSpecBuilder;
+import io.fabric8.kubernetes.api.model.Probe;
+import io.fabric8.kubernetes.api.model.ProbeBuilder;
+import io.fabric8.kubernetes.api.model.Quantity;
+import io.fabric8.kubernetes.api.model.QuantityBuilder;
+import io.fabric8.kubernetes.api.model.ResourceRequirements;
+import io.fabric8.kubernetes.api.model.ResourceRequirementsBuilder;
 
 
 public class KubeUtilTest {
@@ -321,8 +332,16 @@ public class KubeUtilTest {
      * Test for encoding with Go logic.
      */
     @Test
-    public void testSanitizeForGo () {
+    public void testSanitizeForGo1 () {
         assertEquals("as1.telemetryiot-project-ns-4cf002ae-37a5-38d6-9749-b7f85b29a385", KubeUtil.sanitizeForGo("as1", "telemetry/iot-project-ns.iot1"));
     }
 
+    /**
+     * Test for encoding with Go logic (double dash).
+     */
+    @Test
+    public void testSanitizeForGo2 () {
+        assertEquals("managed-address-space.eventiot-project-nsiot-cfb28f7f-66a9-347b-955c-de0a5eeb6cc1", KubeUtil.sanitizeForGo("managed-address-space", "event/iot-project-ns.iot-project-managed"));
+    }
+
 }
diff --git a/iot/iot-device-registry-base/src/main/java/io/enmasse/iot/registry/tenant/KubernetesTenantInformationService.java b/iot/iot-device-registry-base/src/main/java/io/enmasse/iot/registry/tenant/KubernetesTenantInformationService.java
index 69da22cc7..14eec830c 100644
--- a/iot/iot-device-registry-base/src/main/java/io/enmasse/iot/registry/tenant/KubernetesTenantInformationService.java
+++ b/iot/iot-device-registry-base/src/main/java/io/enmasse/iot/registry/tenant/KubernetesTenantInformationService.java
@@ -62,7 +62,7 @@ public class KubernetesTenantInformationService extends AbstractProjectBasedServ
             return failedFuture(e);
         }
 
-        return completedFuture(of(tenantName, tenantName + "/" + project.getMetadata().getCreationTimestamp(), tenant));
+        return completedFuture(of(project.getMetadata().getNamespace(), project.getMetadata().getName(), tenantName + "/" + project.getMetadata().getCreationTimestamp(), tenant));
     }
 
     /**
diff --git a/metrics-api/src/test/java/io/enmasse/metrics/api/PrometheusMetricsFormatterTest.java b/metrics-api/src/test/java/io/enmasse/metrics/api/PrometheusMetricsFormatterTest.java
index bf7268958..2599ff618 100644
--- a/metrics-api/src/test/java/io/enmasse/metrics/api/PrometheusMetricsFormatterTest.java
+++ b/metrics-api/src/test/java/io/enmasse/metrics/api/PrometheusMetricsFormatterTest.java
@@ -23,7 +23,7 @@ public class PrometheusMetricsFormatterTest {
 
         String value = formatter.format(Collections.singletonList(metric), 3);
         System.out.println(value);
-        String expected = "# HELP enmasse_m1 mdesc\n# TYPE enmasse_m1 gauge\nenmasse_m1{key1=\"value1\",key2=\"\\\\this\\\"is\\nescaped\\\\\"} 2 3\n";
+        String expected = "# HELP m1 mdesc\n# TYPE m1 gauge\nm1{key1=\"value1\",key2=\"\\\\this\\\"is\\nescaped\\\\\"} 2 3\n";
         assertEquals(expected, value);
     }
 
diff --git a/systemtests/src/main/java/io/enmasse/systemtest/iot/DeviceRegistryClient.java b/systemtests/src/main/java/io/enmasse/systemtest/iot/DeviceRegistryClient.java
index 42cf3e5da..4c2c35a44 100644
--- a/systemtests/src/main/java/io/enmasse/systemtest/iot/DeviceRegistryClient.java
+++ b/systemtests/src/main/java/io/enmasse/systemtest/iot/DeviceRegistryClient.java
@@ -12,6 +12,7 @@ import org.eclipse.hono.service.management.device.Device;
 import io.enmasse.systemtest.Endpoint;
 import io.vertx.core.buffer.Buffer;
 import io.vertx.core.http.HttpMethod;
+import io.vertx.core.json.DecodeException;
 import io.vertx.core.json.Json;
 import io.vertx.ext.web.client.HttpResponse;
 
@@ -48,7 +49,11 @@ public class DeviceRegistryClient extends HonoApiClient {
         if (result == null) {
             return null;
         }
-        return Json.decodeValue(result, Device.class);
+        try {
+            return Json.decodeValue(result, Device.class);
+        } catch (DecodeException de) {
+            return null;
+        }
     }
 
     public void updateDeviceRegistration(String tenantId, String deviceId, Device payload) throws Exception {
diff --git a/systemtests/src/main/java/io/enmasse/systemtest/operator/OperatorManager.java b/systemtests/src/main/java/io/enmasse/systemtest/operator/OperatorManager.java
index 54607b806..f016359b4 100644
--- a/systemtests/src/main/java/io/enmasse/systemtest/operator/OperatorManager.java
+++ b/systemtests/src/main/java/io/enmasse/systemtest/operator/OperatorManager.java
@@ -267,8 +267,8 @@ public class OperatorManager {
     }
 
     private void cleanCRDs() {
-        KubeCMDClient.runOnCluster("delete", "-v", "6", "crd", "-l", "app=enmasse,enmasse-component=iot");
-        KubeCMDClient.runOnCluster("delete", "-v", "6", "crd", "-l", "app=enmasse");
+        KubeCMDClient.runOnCluster("delete", "crd", "-l", "app=enmasse,enmasse-component=iot");
+        KubeCMDClient.runOnCluster("delete", "crd", "-l", "app=enmasse");
     }
 
     public void waitUntilOperatorReadyOlm() throws Exception {
diff --git a/systemtests/src/main/java/io/enmasse/systemtest/utils/TestUtils.java b/systemtests/src/main/java/io/enmasse/systemtest/utils/TestUtils.java
index 38e4417e0..ac3307d31 100644
--- a/systemtests/src/main/java/io/enmasse/systemtest/utils/TestUtils.java
+++ b/systemtests/src/main/java/io/enmasse/systemtest/utils/TestUtils.java
@@ -5,35 +5,6 @@
 
 package io.enmasse.systemtest.utils;
 
-import com.google.common.io.BaseEncoding;
-import io.enmasse.address.model.Address;
-import io.enmasse.address.model.AddressSpace;
-import io.enmasse.address.model.BrokerState;
-import io.enmasse.address.model.BrokerStatus;
-import io.enmasse.config.AnnotationKeys;
-import io.enmasse.systemtest.Endpoint;
-import io.enmasse.systemtest.Environment;
-import io.enmasse.systemtest.logs.CustomLogger;
-import io.enmasse.systemtest.logs.GlobalLogCollector;
-import io.enmasse.systemtest.model.addressspace.AddressSpaceType;
-import io.enmasse.systemtest.platform.Kubernetes;
-import io.enmasse.systemtest.platform.apps.SystemtestsKubernetesApps;
-import io.enmasse.systemtest.time.SystemtestsOperation;
-import io.enmasse.systemtest.time.TimeMeasuringSystem;
-import io.enmasse.systemtest.time.TimeoutBudget;
-import io.enmasse.systemtest.time.WaitPhase;
-import io.fabric8.kubernetes.api.model.ContainerStatus;
-import io.fabric8.kubernetes.api.model.PersistentVolumeClaim;
-import io.fabric8.kubernetes.api.model.Pod;
-import org.junit.jupiter.api.extension.ExtensionContext;
-import org.junit.jupiter.api.function.ThrowingSupplier;
-import org.openqa.selenium.Capabilities;
-import org.openqa.selenium.chrome.ChromeOptions;
-import org.openqa.selenium.firefox.FirefoxOptions;
-import org.openqa.selenium.remote.RemoteWebDriver;
-import org.opentest4j.AssertionFailedError;
-import org.slf4j.Logger;
-
 import java.net.Inet4Address;
 import java.net.InetAddress;
 import java.net.URL;
@@ -58,6 +29,37 @@ import java.util.function.Supplier;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.jupiter.api.function.ThrowingSupplier;
+import org.openqa.selenium.Capabilities;
+import org.openqa.selenium.chrome.ChromeOptions;
+import org.openqa.selenium.firefox.FirefoxOptions;
+import org.openqa.selenium.remote.RemoteWebDriver;
+import org.opentest4j.AssertionFailedError;
+import org.slf4j.Logger;
+
+import com.google.common.io.BaseEncoding;
+
+import io.enmasse.address.model.Address;
+import io.enmasse.address.model.AddressSpace;
+import io.enmasse.address.model.BrokerState;
+import io.enmasse.address.model.BrokerStatus;
+import io.enmasse.config.AnnotationKeys;
+import io.enmasse.systemtest.Endpoint;
+import io.enmasse.systemtest.Environment;
+import io.enmasse.systemtest.logs.CustomLogger;
+import io.enmasse.systemtest.logs.GlobalLogCollector;
+import io.enmasse.systemtest.model.addressspace.AddressSpaceType;
+import io.enmasse.systemtest.platform.Kubernetes;
+import io.enmasse.systemtest.platform.apps.SystemtestsKubernetesApps;
+import io.enmasse.systemtest.time.SystemtestsOperation;
+import io.enmasse.systemtest.time.TimeMeasuringSystem;
+import io.enmasse.systemtest.time.TimeoutBudget;
+import io.enmasse.systemtest.time.WaitPhase;
+import io.fabric8.kubernetes.api.model.ContainerStatus;
+import io.fabric8.kubernetes.api.model.PersistentVolumeClaim;
+import io.fabric8.kubernetes.api.model.Pod;
+
 public class TestUtils {
 
     public interface TimeoutHandler<X extends Throwable> {
@@ -457,6 +459,7 @@ public class TestUtils {
      * Wait for a condition, fail otherwise.
      *
      * @param condition              The condition to check, returning {@code true} means success.
+     * @param timeout                The maximum time to wait for
      * @param delay                  The delay between checks.
      * @param timeoutMessageSupplier The supplier of a timeout message.
      * @throws AssertionFailedError In case the timeout expired
@@ -473,6 +476,7 @@ public class TestUtils {
      * Wait for a condition, throw exception otherwise.
      *
      * @param condition         The condition to check, returning {@code true} means success.
+     * @param timeout           The maximum time to wait for
      * @param delay             The delay between checks.
      * @param exceptionSupplier The supplier of the exception to throw.
      * @throws AssertionFailedError In case the timeout expired
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/iot/isolated/IoTProjectManagedTest.java b/systemtests/src/test/java/io/enmasse/systemtest/iot/isolated/IoTProjectManagedTest.java
index 7d476d9d4..820d3d862 100644
--- a/systemtests/src/test/java/io/enmasse/systemtest/iot/isolated/IoTProjectManagedTest.java
+++ b/systemtests/src/test/java/io/enmasse/systemtest/iot/isolated/IoTProjectManagedTest.java
@@ -1,59 +1,272 @@
 /*
- * Copyright 2018-2019, EnMasse authors.
+ * Copyright 2018-2020, EnMasse authors.
  * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html).
  */
 package io.enmasse.systemtest.iot.isolated;
 
+import static io.enmasse.systemtest.TestTag.ACCEPTANCE;
+import static io.enmasse.systemtest.TestTag.SMOKE;
+import static io.enmasse.systemtest.time.TimeoutBudget.ofDuration;
+import static io.enmasse.user.model.v1.Operation.recv;
+import static io.enmasse.user.model.v1.Operation.send;
+import static java.time.Duration.ofMinutes;
+import static java.util.Arrays.asList;
+import static java.util.EnumSet.of;
+import static java.util.Optional.ofNullable;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.beans.HasPropertyWithValue.hasProperty;
+import static org.hamcrest.collection.IsCollectionWithSize.hasSize;
+import static org.hamcrest.collection.IsIterableContainingInAnyOrder.containsInAnyOrder;
+import static org.hamcrest.core.AllOf.allOf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.time.Duration;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.hamcrest.Matcher;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+
 import io.enmasse.address.model.Address;
 import io.enmasse.address.model.AddressSpace;
+import io.enmasse.address.model.AddressSpaceStatus;
+import io.enmasse.address.model.AddressStatus;
+import io.enmasse.address.model.KubeUtil;
+import io.enmasse.address.model.Phase;
+import io.enmasse.iot.model.v1.DoneableIoTProject;
 import io.enmasse.iot.model.v1.IoTConfigBuilder;
 import io.enmasse.iot.model.v1.IoTProject;
-import io.enmasse.systemtest.TestTag;
+import io.enmasse.iot.model.v1.IoTProjectList;
 import io.enmasse.systemtest.bases.TestBase;
 import io.enmasse.systemtest.bases.iot.ITestIoTIsolated;
+import io.enmasse.systemtest.logs.CustomLogger;
 import io.enmasse.systemtest.model.address.AddressType;
 import io.enmasse.systemtest.model.addressplan.DestinationPlan;
 import io.enmasse.systemtest.model.addressspace.AddressSpacePlans;
 import io.enmasse.systemtest.model.addressspace.AddressSpaceType;
 import io.enmasse.systemtest.platform.Kubernetes;
+import io.enmasse.systemtest.time.TimeoutBudget;
 import io.enmasse.systemtest.utils.AddressUtils;
 import io.enmasse.systemtest.utils.IoTUtils;
+import io.enmasse.systemtest.utils.TestUtils;
 import io.enmasse.user.model.v1.Operation;
 import io.enmasse.user.model.v1.User;
 import io.enmasse.user.model.v1.UserAuthorization;
+import io.enmasse.user.model.v1.UserStatus;
+import io.fabric8.kubernetes.api.model.Doneable;
+import io.fabric8.kubernetes.api.model.HasMetadata;
 import io.fabric8.kubernetes.api.model.ObjectMeta;
 import io.fabric8.kubernetes.api.model.OwnerReference;
-import org.hamcrest.Matcher;
-import org.junit.jupiter.api.Tag;
-import org.junit.jupiter.api.Test;
+import io.fabric8.kubernetes.client.dsl.MixedOperation;
+import io.fabric8.kubernetes.client.dsl.Resource;
 
-import java.util.Arrays;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import static io.enmasse.systemtest.TestTag.ACCEPTANCE;
-import static io.enmasse.user.model.v1.Operation.recv;
-import static io.enmasse.user.model.v1.Operation.send;
-import static java.util.Arrays.asList;
-import static java.util.EnumSet.of;
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.beans.HasPropertyWithValue.hasProperty;
-import static org.hamcrest.collection.IsCollectionWithSize.hasSize;
-import static org.hamcrest.collection.IsIterableContainingInAnyOrder.containsInAnyOrder;
-import static org.hamcrest.core.AllOf.allOf;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
-@Tag(TestTag.SMOKE)
 class IoTProjectManagedTest extends TestBase implements ITestIoTIsolated {
+    private final static Logger log = CustomLogger.getLogger();
+
+    @FunctionalInterface
+    interface ProjectModificator {
+        /**
+         * Modify a project
+         *
+         * @param timeout The timeout budget the operation has available.
+         * @param project The project to work with. It is an provisioned project, which already passed a
+         *        call to {@link IoTProjectManagedTest#assertManaged(IoTProject)}.
+         * @return {@code true} if the state has been change and a final call to
+         *         {@link IoTProjectManagedTest#assertManaged(IoTProject)} to be performed.
+         */
+        boolean modify(TimeoutBudget timeout, IoTProject project) throws Exception;
+    }
+
     private Kubernetes kubernetes = Kubernetes.getInstance();
 
+    private MixedOperation<IoTProject, IoTProjectList, DoneableIoTProject, Resource<IoTProject, DoneableIoTProject>> projectClient;
+
+    @BeforeEach
+    void createIoTClient() {
+        this.projectClient = this.kubernetes.getIoTProjectClient(IOT_PROJECT_NAMESPACE);
+    }
+
+    /**
+     * Simply create a project and directly test it.
+     */
     @Test
+    @Tag(SMOKE)
     @Tag(ACCEPTANCE)
     void testCreate() throws Exception {
+        // run a default test, with no modifications
+        doTestAddressSpaceWithModifications(ofDuration(ofMinutes(5)), (timeout, project) -> false);
+    }
+
+    /**
+     * Delete a resource and wait for it to be re-created properly.
+     *
+     * @param <T> The resource type.
+     * @param <LT> The list type of the resource.
+     * @param <D> The {@link Doneable} type.
+     * @param project The IoT project to process.
+     * @param clazz The class instance of the resource type.
+     * @param name The kubernetes name of the resource.
+     * @param basicClient The basic resource client. Does not need to be namespaced.
+     * @param phaseExtractor The function which extracts the phase information.
+     */
+    <T extends HasMetadata, LT, D> ProjectModificator deleteAndWaitResource(final Class<T> clazz, final Function<IoTProject, String> nameExtractor,
+            final MixedOperation<T, LT, D, Resource<T, D>> basicClient, final Function<T, Optional<Phase>> phaseExtractor) {
+
+        return (timeout, project) -> {
+
+            final var name = nameExtractor.apply(project);
+            final var className = clazz.getSimpleName();
+            final var client = basicClient.inNamespace(IOT_PROJECT_NAMESPACE);
+
+            // get the current address space
+
+            final var namedClient = client.withName(name);
+            final var currentResource = namedClient.get();
+            assertNotNull(currentResource, () -> String.format("Unable to find resource - type: %s, name: '%s'", className, name));
+
+            // remember its ID
+
+            final String originalId = currentResource.getMetadata().getUid();
+            assertNotNull(originalId);
+
+            // delete it
+
+            client.withName(name).delete();
+
+            // now wait for it to be re-created
+
+            TestUtils.waitUntilConditionOrFail(() -> {
+
+                var current = namedClient.get();
+                if (current == null) {
+                    log.info("{} is still missing", className);
+                    return false;
+                }
+                if (originalId.equals(current.getMetadata().getUid())) {
+                    log.info("{} has still the same ID", className);
+                    // still the same object
+                    return false;
+                }
+
+                // get current phase
+                final var phase = phaseExtractor.apply(current);
+                if (!phase.isPresent()) {
+                    log.info("{} no phase information", className);
+                    return false;
+                }
+
+                if (phase.get() != Phase.Active) {
+                    log.info("{} is not ready yet: {}", className, phase.get());
+                    return false;
+                }
+
+                return true;
+            }, timeout.remaining(), Duration.ofSeconds(10), () -> className + " did not get re-created");
+
+            // and wait for the IoT project to become ready again
+
+            var projectAccess = kubernetes.getIoTProjectClient(project.getMetadata().getNamespace()).withName(project.getMetadata().getName());
+            TestUtils.waitUntilConditionOrFail(() -> {
+                var current = projectAccess.get();
+
+                if (current == null) {
+                    log.info("IoTProject missing");
+                    return false;
+                }
+
+                if (current.getStatus() == null || current.getStatus().getPhase() == null) {
+                    log.info("IoTProject is missing status information");
+                    return false;
+                }
+
+                if (!current.getStatus().getPhase().equals("Active")) {
+                    return false;
+                }
+
+                return true;
+            }, timeout.remaining(), Duration.ofSeconds(10), () -> "IoTProject failed to switch back to 'Active'");
+
+            return true;
+        };
+    }
+
+    /**
+     * Test deleting the whole address space.
+     */
+    @Test
+    void testDeleteAddressSpace() throws Exception {
+        doTestAddressSpaceWithModifications(
+                ofDuration(ofMinutes(15)),
+                deleteAndWaitResource(
+                        AddressSpace.class,
+                        project -> project.getSpec().getDownstreamStrategy().getManagedStrategy().getAddressSpace().getName(),
+                        kubernetes.getAddressSpaceClient(),
+                        addressSpace -> ofNullable(addressSpace).map(AddressSpace::getStatus).map(AddressSpaceStatus::getPhase)));
+    }
+
+    /**
+     * Test deleting the telememtry address.
+     */
+    @Test
+    void testDeleteTelemetryAddress() throws Exception {
+        doTestAddressSpaceWithModifications(
+                ofDuration(ofMinutes(10)),
+                deleteAndWaitResource(
+                        Address.class,
+                        project -> KubeUtil.sanitizeForGo(
+                                project.getSpec().getDownstreamStrategy().getManagedStrategy().getAddressSpace().getName(),
+                                "telemetry/" + project.getStatus().getTenantName()),
+                        kubernetes.getAddressClient(),
+                        address -> ofNullable(address).map(Address::getStatus).map(AddressStatus::getPhase)));
+    }
+
+    /**
+     * Test deleting the event address.
+     * <br/>
+     * The event address is expected to be backed by a brokered address. Thus is might behave
+     * differently than the others.
+     */
+    @Test
+    void testDeleteEventAddress() throws Exception {
+        doTestAddressSpaceWithModifications(
+                ofDuration(ofMinutes(15)),
+                deleteAndWaitResource(
+                        Address.class,
+                        project -> KubeUtil.sanitizeForGo(
+                                project.getSpec().getDownstreamStrategy().getManagedStrategy().getAddressSpace().getName(),
+                                "event/" + project.getStatus().getTenantName()),
+                        kubernetes.getAddressClient(),
+                        address -> ofNullable(address).map(Address::getStatus).map(AddressStatus::getPhase)));
+    }
+
+    /**
+     * Test deleting the adapter user.
+     */
+    @Test
+    void testDeleteAdapterUser() throws Exception {
+        doTestAddressSpaceWithModifications(
+                ofDuration(ofMinutes(10)),
+                deleteAndWaitResource(
+                        User.class,
+                        project -> project.getSpec().getDownstreamStrategy().getManagedStrategy().getAddressSpace().getName()
+                                + "."
+                                + project.getStatus().getDownstreamEndpoint().getUsername(),
+                        kubernetes.getUserClient(),
+                        user -> ofNullable(user).map(User::getStatus).map(UserStatus::getPhase)));
+    }
+
+    void doTestAddressSpaceWithModifications(final TimeoutBudget timeout, final ProjectModificator modificator) throws Exception {
+
         isolatedIoTManager.createIoTConfig(new IoTConfigBuilder()
                 .withNewMetadata()
                 .withName("default")
@@ -70,14 +283,14 @@ class IoTProjectManagedTest extends TestBase implements ITestIoTIsolated {
                 .endSpec()
                 .build());
 
-        String addressSpaceName = "managed-address-space";
+        final String addressSpaceName = "managed-address-space";
+        final String iotProjectName = "iot-project-managed";
 
-        IoTProject project = IoTUtils.getBasicIoTProjectObject("iot-project-managed", addressSpaceName,
+        IoTProject project = IoTUtils.getBasicIoTProjectObject(iotProjectName, addressSpaceName,
                 IOT_PROJECT_NAMESPACE, getDefaultAddressSpacePlan());
         LOGGER.warn("NAMESPACE EXISTS? {}, {}", project.getMetadata().getNamespace(), kubernetes.namespaceExists(project.getMetadata().getNamespace()));
-        isolatedIoTManager.createIoTProject(project);// waiting until ready
-        var iotProjectApiClient = kubernetes.getIoTProjectClient(project.getMetadata().getNamespace());
-        IoTProject created = iotProjectApiClient.withName(project.getMetadata().getName()).get();
+        isolatedIoTManager.createIoTProject(project); // waiting until ready
+        IoTProject created = this.projectClient.withName(iotProjectName).get();
 
         assertNotNull(created);
         assertEquals(IOT_PROJECT_NAMESPACE, created.getMetadata().getNamespace());
@@ -88,19 +301,26 @@ class IoTProjectManagedTest extends TestBase implements ITestIoTIsolated {
 
         assertManaged(created);
 
+        if (modificator.modify(timeout, created)) {
+            created = this.projectClient.withName(iotProjectName).get();
+            assertNotNull(created);
+            assertManaged(created);
+        }
     }
 
-    private static void assertAddressType (final Address address, final AddressType type, final String plan) {
+    private static void assertAddressType(final Address address, final AddressType type, final String plan) {
         assertEquals(type.toString(), address.getSpec().getType());
         assertEquals(plan, address.getSpec().getPlan());
     }
 
     private void assertManaged(IoTProject project) throws Exception {
-        //address spaces
-        AddressSpace addressSpace = isolatedIoTManager.getAddressSpace(IOT_PROJECT_NAMESPACE, project.getSpec().getDownstreamStrategy().getManagedStrategy().getAddressSpace().getName());
+        // address spaces
+        AddressSpace addressSpace =
+                isolatedIoTManager.getAddressSpace(IOT_PROJECT_NAMESPACE, project.getSpec().getDownstreamStrategy().getManagedStrategy().getAddressSpace().getName());
         assertEquals(project.getSpec().getDownstreamStrategy().getManagedStrategy().getAddressSpace().getName(), addressSpace.getMetadata().getName());
         assertEquals(AddressSpaceType.STANDARD.toString(), addressSpace.getSpec().getType());
         assertEquals(AddressSpacePlans.STANDARD_SMALL, addressSpace.getSpec().getPlan());
+        assertEquals(Phase.Active, addressSpace.getStatus().getPhase());
 
         // addresses
         // {event/control/command/command_response/telemetry}/"project-namespace"."project-name"
@@ -136,11 +356,13 @@ class IoTProjectManagedTest extends TestBase implements ITestIoTIsolated {
 
             }
 
+            assertEquals(Phase.Active, address.getStatus().getPhase());
+
         }
         assertEquals(5, correctAddressesCounter, "There are incorrect IoT addresses " + addresses);
 
-        //username "adapter"
-        //name "project-address-space"+".adapter"
+        // username "adapter"
+        // name "project-address-space"+".adapter"
         User user = isolatedIoTManager.getUser(addressSpace, "adapter-" + project.getMetadata().getUid());
         assertNotNull(user);
         assertEquals(1, user.getMetadata().getOwnerReferences().size());
@@ -152,9 +374,9 @@ class IoTProjectManagedTest extends TestBase implements ITestIoTIsolated {
 
         assertThat(authorizations, containsInAnyOrder(
                 asList(
-                        assertAdapterAuthorization( of(send), expandAddresses(addressSuffix, IOT_ADDRESS_TELEMETRY, IOT_ADDRESS_EVENT, IOT_ADDRESS_COMMAND_RESPONSE)),
-                        assertAdapterAuthorization( of(recv), expandAddresses(addressSuffix, IOT_ADDRESS_COMMAND)),
-                        assertAdapterAuthorization( of(recv, send), expandAddresses(addressSuffix, IOT_ADDRESS_CONTROL)))));
+                        assertAdapterAuthorization(of(send), expandAddresses(addressSuffix, IOT_ADDRESS_TELEMETRY, IOT_ADDRESS_EVENT, IOT_ADDRESS_COMMAND_RESPONSE)),
+                        assertAdapterAuthorization(of(recv), expandAddresses(addressSuffix, IOT_ADDRESS_COMMAND)),
+                        assertAdapterAuthorization(of(recv, send), expandAddresses(addressSuffix, IOT_ADDRESS_CONTROL)))));
     }
 
     /**
@@ -197,6 +419,7 @@ class IoTProjectManagedTest extends TestBase implements ITestIoTIsolated {
 
     /**
      * Test if the project is the owner the reference points to.
+     *
      * @param project The project to check for.
      * @param ownerReference The reference to check.
      * @return {@code true} if the reference points to the project.
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/iot/isolated/registry/DeviceRegistryTest.java b/systemtests/src/test/java/io/enmasse/systemtest/iot/isolated/registry/DeviceRegistryTest.java
index b85db9656..fc0a3a14a 100644
--- a/systemtests/src/test/java/io/enmasse/systemtest/iot/isolated/registry/DeviceRegistryTest.java
+++ b/systemtests/src/test/java/io/enmasse/systemtest/iot/isolated/registry/DeviceRegistryTest.java
@@ -43,7 +43,6 @@ abstract class DeviceRegistryTest extends TestBase implements ITestIoTIsolated {
 
     private static final String DEVICE_REGISTRY_TEST_PROJECT = "device-registry-test-project";
 
-
     private String randomDeviceId;
     private IoTConfig iotConfig;
     private IoTProject iotProject;
@@ -57,6 +56,8 @@ abstract class DeviceRegistryTest extends TestBase implements ITestIoTIsolated {
 
     private AmqpClient amqpClient;
 
+    protected static int TENANT_DOESNT_EXISTS_CODE = HTTP_NOT_FOUND;
+
     protected abstract IoTConfigBuilder provideIoTConfig() throws Exception;
 
     @BeforeEach
@@ -268,7 +269,7 @@ abstract class DeviceRegistryTest extends TestBase implements ITestIoTIsolated {
 
     protected void doTestCreateForNonExistingTenantFails() throws Exception {
         var response = client.registerDeviceWithResponse("invalid-" + isolatedIoTManager.getTenantId(), randomDeviceId);
-        assertEquals(HTTP_NOT_FOUND, response.statusCode());
+        assertEquals(TENANT_DOESNT_EXISTS_CODE, response.statusCode());
     }
 
     protected void doTestTenantDeletionTriggersDevicesDeletion() throws Exception {
@@ -292,7 +293,7 @@ abstract class DeviceRegistryTest extends TestBase implements ITestIoTIsolated {
 
             // second check, the credentials and device should be deleted
 
-            client.getDeviceRegistration(tenantId, randomDeviceId, HttpURLConnection.HTTP_NOT_FOUND);
+            client.getDeviceRegistration(tenantId, randomDeviceId, TENANT_DOESNT_EXISTS_CODE);
             IoTUtils.checkCredentials(authId, newPassword, true, httpAdapterEndpoint, amqpClient, iotProject);
         }
     }
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/iot/isolated/registry/InfinispanDeviceRegistryTest.java b/systemtests/src/test/java/io/enmasse/systemtest/iot/isolated/registry/InfinispanDeviceRegistryTest.java
index 8f15d4bfb..3fc796f64 100644
--- a/systemtests/src/test/java/io/enmasse/systemtest/iot/isolated/registry/InfinispanDeviceRegistryTest.java
+++ b/systemtests/src/test/java/io/enmasse/systemtest/iot/isolated/registry/InfinispanDeviceRegistryTest.java
@@ -11,8 +11,14 @@ import org.junit.jupiter.api.Test;
 import static io.enmasse.systemtest.iot.DefaultDeviceRegistry.newInfinispanBased;
 import static io.enmasse.systemtest.utils.IoTUtils.assertCorrectRegistryType;
 
+import java.net.HttpURLConnection;
+
 class InfinispanDeviceRegistryTest extends DeviceRegistryTest {
 
+    static {
+        TENANT_DOESNT_EXISTS_CODE = HttpURLConnection.HTTP_UNAUTHORIZED;
+    }
+
     @Override
     protected IoTConfigBuilder provideIoTConfig() throws Exception {
         return new IoTConfigBuilder()
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/isolated/CommonTest.java b/systemtests/src/test/java/io/enmasse/systemtest/isolated/CommonTest.java
index d1bf63cd2..385ab914f 100644
--- a/systemtests/src/test/java/io/enmasse/systemtest/isolated/CommonTest.java
+++ b/systemtests/src/test/java/io/enmasse/systemtest/isolated/CommonTest.java
@@ -98,7 +98,7 @@ class CommonTest extends TestBase implements ITestBaseIsolated {
 
         Multimap<String, String> podsContainersWithNoLog = HashMultimap.create();
 
-        kubernetes.listPods().forEach(pod -> kubernetes.getContainersFromPod(pod.getMetadata().getName()).forEach(container -> {
+        kubernetes.listPods().stream().filter(pod -> !pod.getMetadata().getName().contains("none-authservice")).forEach(pod -> kubernetes.getContainersFromPod(pod.getMetadata().getName()).forEach(container -> {
             String podName = pod.getMetadata().getName();
             String containerName = container.getName();
             log.info("Getting log from pod: {}, for container: {}", podName, containerName);
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/isolated/monitoring/MonitoringTest.java b/systemtests/src/test/java/io/enmasse/systemtest/isolated/monitoring/MonitoringTest.java
index e353b641e..8334e3724 100644
--- a/systemtests/src/test/java/io/enmasse/systemtest/isolated/monitoring/MonitoringTest.java
+++ b/systemtests/src/test/java/io/enmasse/systemtest/isolated/monitoring/MonitoringTest.java
@@ -105,7 +105,7 @@ class MonitoringTest extends TestBase implements ITestIsolatedStandard {
                 }
             }
             return false;
-        }, new TimeoutBudget(3, TimeUnit.MINUTES));
+        }, new TimeoutBudget(10, TimeUnit.MINUTES));
 
     }