Repository: EnMasseProject/enmasse
Pull request #2685 (2019-04-16 14:56:24)

recommend standard openshift user for the tests
Running the tests as system:admin may break the tests, i lost a fair bit of time with this :)


Patch:
diff --git a/address-space-controller/src/main/java/io/enmasse/controller/AddressSpaceController.java b/address-space-controller/src/main/java/io/enmasse/controller/AddressSpaceController.java
index 6e11ad09b..d1bb84ba5 100644
--- a/address-space-controller/src/main/java/io/enmasse/controller/AddressSpaceController.java
+++ b/address-space-controller/src/main/java/io/enmasse/controller/AddressSpaceController.java
@@ -9,14 +9,13 @@ import io.enmasse.admin.model.v1.AuthenticationServiceType;
 import io.enmasse.controller.auth.*;
 import io.enmasse.controller.common.Kubernetes;
 import io.enmasse.controller.common.KubernetesHelper;
-import io.enmasse.controller.keycloak.Keycloak;
 import io.enmasse.controller.keycloak.RealmController;
 import io.enmasse.k8s.api.*;
 import io.enmasse.metrics.api.Metrics;
 import io.enmasse.model.CustomResourceDefinitions;
+import io.enmasse.user.api.DelegateUserApi;
 import io.enmasse.user.api.NullUserApi;
 import io.enmasse.user.api.UserApi;
-import io.enmasse.user.api.DelegateUserApi;
 import io.enmasse.user.keycloak.KeycloakFactory;
 import io.enmasse.user.keycloak.KeycloakUserApi;
 import io.enmasse.user.keycloak.KubeKeycloakFactory;
@@ -97,16 +96,18 @@ public class AddressSpaceController {
 
         InfraResourceFactory infraResourceFactory = new TemplateInfraResourceFactory(kubernetes, authenticationServiceRegistry, System.getenv(), isOpenShift, schemaProvider);
 
-        KeycloakFactory keycloakFactory = new KubeKeycloakFactory(controllerClient);
         Clock clock = Clock.systemUTC();
+        KeycloakFactory keycloakFactory = new KubeKeycloakFactory(controllerClient);
+        KeycloakUserApi keycloakUserApi = new KeycloakUserApi(keycloakFactory, clock, Duration.ZERO);
+        schemaProvider.registerListener(newSchema -> keycloakUserApi.retainAuthenticationServices(newSchema.findAuthenticationServiceType(AuthenticationServiceType.standard)));
         UserApi userApi = new DelegateUserApi(Map.of(AuthenticationServiceType.none, new NullUserApi(),
                 AuthenticationServiceType.external, new NullUserApi(),
-                AuthenticationServiceType.standard, new KeycloakUserApi(keycloakFactory, clock, Duration.ZERO)));
+                AuthenticationServiceType.standard, keycloakUserApi));
 
         Metrics metrics = new Metrics();
         controllerChain = new ControllerChain(kubernetes, addressSpaceApi, schemaProvider, eventLogger, metrics, options.getVersion(), options.getRecheckInterval(), options.getResyncInterval());
         controllerChain.addController(new CreateController(kubernetes, schemaProvider, infraResourceFactory, eventLogger, authController.getDefaultCertProvider(), options.getVersion(), addressSpaceApi));
-        controllerChain.addController(new RealmController(new Keycloak(keycloakFactory), authenticationServiceRegistry));
+        controllerChain.addController(new RealmController(keycloakUserApi, authenticationServiceRegistry));
         controllerChain.addController(new NetworkPolicyController(controllerClient, schemaProvider));
         controllerChain.addController(new StatusController(kubernetes, schemaProvider, infraResourceFactory, authenticationServiceRegistry, userApi));
         controllerChain.addController(new EndpointController(controllerClient, options.isExposeEndpointsByDefault(), isOpenShift));
diff --git a/address-space-controller/src/main/java/io/enmasse/controller/TemplateInfraResourceFactory.java b/address-space-controller/src/main/java/io/enmasse/controller/TemplateInfraResourceFactory.java
index 8613a1cd4..b40bd748e 100644
--- a/address-space-controller/src/main/java/io/enmasse/controller/TemplateInfraResourceFactory.java
+++ b/address-space-controller/src/main/java/io/enmasse/controller/TemplateInfraResourceFactory.java
@@ -6,11 +6,10 @@ package io.enmasse.controller;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import io.enmasse.address.model.AddressSpace;
-import io.enmasse.address.model.CertSpec;
-import io.enmasse.address.model.EndpointSpec;
-import io.enmasse.address.model.KubeUtil;
+import io.enmasse.address.model.*;
 import io.enmasse.admin.model.v1.*;
+import io.enmasse.admin.model.v1.AuthenticationService;
+import io.enmasse.admin.model.v1.AuthenticationServiceType;
 import io.enmasse.config.AnnotationKeys;
 import io.enmasse.config.LabelKeys;
 import io.enmasse.controller.common.Kubernetes;
@@ -70,14 +69,17 @@ public class TemplateInfraResourceFactory implements InfraResourceFactory {
         String authServiceRealm = authService.getSpec().getRealm() != null ? authService.getSpec().getRealm() : addressSpace.getAnnotation(AnnotationKeys.REALM_NAME);
 
         if (authService.getSpec().getType().equals(AuthenticationServiceType.external) && authService.getSpec().getExternal() != null && authService.getSpec().getExternal().isAllowOverride()) {
-            if (addressSpace.getSpec().getAuthenticationService().getHost() != null) {
-                authServiceHost = addressSpace.getSpec().getAuthenticationService().getHost();
-            }
-            if (addressSpace.getSpec().getAuthenticationService().getPort() != null) {
-                authServicePort = addressSpace.getSpec().getAuthenticationService().getPort();
-            }
-            if (addressSpace.getSpec().getAuthenticationService().getRealm() != null) {
-                authServiceRealm = addressSpace.getSpec().getAuthenticationService().getRealm();
+            if (addressSpace.getSpec().getAuthenticationService().getOverrides() != null) {
+                AuthenticationServiceOverrides overrides = addressSpace.getSpec().getAuthenticationService().getOverrides();
+                if (overrides.getHost() != null) {
+                    authServiceHost = overrides.getHost();
+                }
+                if (overrides.getPort() != null) {
+                    authServicePort = overrides.getPort();
+                }
+                if (overrides.getRealm() != null) {
+                    authServiceRealm = overrides.getRealm();
+                }
             }
         }
 
diff --git a/address-space-controller/src/main/java/io/enmasse/controller/keycloak/Keycloak.java b/address-space-controller/src/main/java/io/enmasse/controller/keycloak/Keycloak.java
deleted file mode 100644
index 5c530bc10..000000000
--- a/address-space-controller/src/main/java/io/enmasse/controller/keycloak/Keycloak.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Copyright 2017-2018, EnMasse authors.
- * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html).
- */
-package io.enmasse.controller.keycloak;
-
-import io.enmasse.admin.model.v1.AuthenticationService;
-import io.enmasse.user.keycloak.KeycloakFactory;
-import org.keycloak.representations.idm.RealmRepresentation;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-public class Keycloak implements KeycloakApi {
-
-    private final KeycloakFactory keycloakFactory;
-    private final Map<String, org.keycloak.admin.client.Keycloak> keycloakMap = new HashMap<>();
-
-    public Keycloak(KeycloakFactory keycloakFactory) {
-        this.keycloakFactory = keycloakFactory;
-    }
-
-    interface Handler<T> {
-        T handle(org.keycloak.admin.client.Keycloak keycloak);
-    }
-
-    private synchronized <T> T withKeycloak(AuthenticationService authenticationService, Handler<T> consumer) {
-        if (keycloakMap.get(authenticationService.getMetadata().getName()) == null) {
-            keycloakMap.put(authenticationService.getMetadata().getName(), keycloakFactory.createInstance(authenticationService));
-        }
-        org.keycloak.admin.client.Keycloak keycloak = keycloakMap.get(authenticationService.getMetadata().getName());
-        try {
-            return consumer.handle(keycloak);
-        } catch (Exception e) {
-            keycloakMap.remove(authenticationService.getMetadata().getName());
-            keycloak.close();
-            throw e;
-        }
-    }
-
-    @Override
-    public Set<String> getRealmNames(AuthenticationService authenticationService) {
-        return withKeycloak(authenticationService, kc -> kc.realms().findAll().stream()
-                .map(RealmRepresentation::getRealm)
-                .collect(Collectors.toSet()));
-    }
-
-    @Override
-    public void createRealm(AuthenticationService authenticationService, String namespace, String realmName) {
-        final RealmRepresentation newRealm = new RealmRepresentation();
-        newRealm.setRealm(realmName);
-        newRealm.setEnabled(true);
-        newRealm.setPasswordPolicy("hashAlgorithm(scramsha1)");
-        newRealm.setAttributes(new HashMap<>());
-        newRealm.getAttributes().put("namespace", namespace);
-        newRealm.getAttributes().put("enmasse-realm", "true");
-
-        withKeycloak(authenticationService, kc -> {
-            kc.realms().create(newRealm);
-            return true;
-        });
-    }
-
-    @Override
-    public void deleteRealm(AuthenticationService authenticationService, String realmName) {
-        withKeycloak(authenticationService, kc -> {
-            kc.realm(realmName).remove();
-            return true;
-        });
-    }
-}
diff --git a/address-space-controller/src/main/java/io/enmasse/controller/keycloak/RealmController.java b/address-space-controller/src/main/java/io/enmasse/controller/keycloak/RealmController.java
index d163eeac3..9eee94fb5 100644
--- a/address-space-controller/src/main/java/io/enmasse/controller/keycloak/RealmController.java
+++ b/address-space-controller/src/main/java/io/enmasse/controller/keycloak/RealmController.java
@@ -11,6 +11,7 @@ import io.enmasse.admin.model.v1.AuthenticationServiceType;
 import io.enmasse.config.AnnotationKeys;
 import io.enmasse.controller.Controller;
 import io.enmasse.k8s.api.AuthenticationServiceRegistry;
+import io.enmasse.user.api.RealmApi;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -20,10 +21,10 @@ import java.util.stream.Collectors;
 public class RealmController implements Controller {
     private static final Logger log = LoggerFactory.getLogger(RealmController.class);
     private static final String MASTER_REALM = "master";
-    private final KeycloakApi keycloak;
+    private final RealmApi keycloak;
     private final AuthenticationServiceRegistry authenticationServiceRegistry;
 
-    public RealmController(KeycloakApi keycloak, AuthenticationServiceRegistry authenticationServiceRegistry) {
+    public RealmController(RealmApi keycloak, AuthenticationServiceRegistry authenticationServiceRegistry) {
         this.keycloak = keycloak;
         this.authenticationServiceRegistry = authenticationServiceRegistry;
     }
diff --git a/api-common/pom.xml b/api-common/pom.xml
index e1f1b0839..5c0030ad6 100644
--- a/api-common/pom.xml
+++ b/api-common/pom.xml
@@ -13,6 +13,11 @@
       <groupId>io.enmasse</groupId>
       <artifactId>api-model</artifactId>
     </dependency>
+    <dependency>
+      <groupId>io.enmasse</groupId>
+      <artifactId>keycloak-user-api</artifactId>
+      <scope>compile</scope>
+    </dependency>
     <dependency>
       <groupId>io.enmasse</groupId>
       <artifactId>amqp-utils</artifactId>
diff --git a/api-common/src/main/java/io/enmasse/api/common/DefaultExceptionMapper.java b/api-common/src/main/java/io/enmasse/api/common/DefaultExceptionMapper.java
index 4afd8549b..74f93ffb4 100644
--- a/api-common/src/main/java/io/enmasse/api/common/DefaultExceptionMapper.java
+++ b/api-common/src/main/java/io/enmasse/api/common/DefaultExceptionMapper.java
@@ -13,6 +13,7 @@ import javax.ws.rs.ext.Provider;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import io.enmasse.address.model.UnresolvedAddressException;
 import io.enmasse.address.model.UnresolvedAddressSpaceException;
+import io.enmasse.user.keycloak.KeycloakUnavailableException;
 import io.enmasse.user.model.v1.UserValidationFailedException;
 import io.fabric8.kubernetes.client.KubernetesClientException;
 import org.slf4j.Logger;
@@ -31,6 +32,8 @@ public class DefaultExceptionMapper implements ExceptionMapper<Exception> {
             statusCode = ((KubernetesClientException) exception).getStatus().getCode();
         } else if (exception instanceof UnresolvedAddressException || exception instanceof JsonProcessingException || exception instanceof UnresolvedAddressSpaceException || exception instanceof ValidationException || exception instanceof UserValidationFailedException) {
             statusCode = Response.Status.BAD_REQUEST.getStatusCode();
+        } else if (exception instanceof KeycloakUnavailableException) {
+            statusCode = Response.Status.SERVICE_UNAVAILABLE.getStatusCode();
         } else {
             statusCode = Response.Status.INTERNAL_SERVER_ERROR.getStatusCode();
         }
diff --git a/api-model/src/main/java/io/enmasse/address/model/AuthenticationService.java b/api-model/src/main/java/io/enmasse/address/model/AuthenticationService.java
index a51de0ef1..a7bb692c9 100644
--- a/api-model/src/main/java/io/enmasse/address/model/AuthenticationService.java
+++ b/api-model/src/main/java/io/enmasse/address/model/AuthenticationService.java
@@ -32,9 +32,7 @@ public class AuthenticationService extends AbstractWithAdditionalProperties {
 
     private String name;
 
-    private String host;
-    private Integer port;
-    private String realm;
+    private AuthenticationServiceOverrides overrides;
 
     // TODO: Keep for backwards compatibility
     private AuthenticationServiceType type;
@@ -58,28 +56,12 @@ public class AuthenticationService extends AbstractWithAdditionalProperties {
         this.name = name;
     }
 
-    public String getHost() {
-        return host;
+    public void setOverrides(AuthenticationServiceOverrides overrides) {
+        this.overrides = overrides;
     }
 
-    public void setHost(String host) {
-        this.host = host;
-    }
-
-    public Integer getPort() {
-        return port;
-    }
-
-    public void setPort(Integer port) {
-        this.port = port;
-    }
-
-    public String getRealm() {
-        return realm;
-    }
-
-    public void setRealm(String realm) {
-        this.realm = realm;
+    public AuthenticationServiceOverrides getOverrides() {
+        return overrides;
     }
 
     @Override
@@ -89,13 +71,11 @@ public class AuthenticationService extends AbstractWithAdditionalProperties {
         final AuthenticationService that = (AuthenticationService) o;
         return Objects.equals(name, that.name) &&
                 Objects.equals(type, that.type) &&
-                Objects.equals(host, that.host) &&
-                Objects.equals(port, that.port) &&
-                Objects.equals(realm, that.realm);
+                Objects.equals(overrides, that.overrides);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(name, type, host, port, realm);
+        return Objects.hash(name, type, overrides);
     }
 }
diff --git a/api-model/src/main/java/io/enmasse/address/model/AuthenticationServiceOverrides.java b/api-model/src/main/java/io/enmasse/address/model/AuthenticationServiceOverrides.java
new file mode 100644
index 000000000..25ad0d2c6
--- /dev/null
+++ b/api-model/src/main/java/io/enmasse/address/model/AuthenticationServiceOverrides.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright 2019, EnMasse authors.
+ * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html).
+ */
+package io.enmasse.address.model;
+
+import java.util.Objects;
+
+public class AuthenticationServiceOverrides {
+    private String host;
+    private Integer port;
+    private String realm;
+
+    public String getHost() {
+        return host;
+    }
+
+    public void setHost(String host) {
+        this.host = host;
+    }
+
+    public Integer getPort() {
+        return port;
+    }
+
+    public void setPort(Integer port) {
+        this.port = port;
+    }
+
+    public String getRealm() {
+        return realm;
+    }
+
+    public void setRealm(String realm) {
+        this.realm = realm;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        AuthenticationServiceOverrides that = (AuthenticationServiceOverrides) o;
+        return Objects.equals(host, that.host) &&
+                Objects.equals(port, that.port) &&
+                Objects.equals(realm, that.realm);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(host, port, realm);
+    }
+
+    @Override
+    public String toString() {
+        return "AuthenticationServiceOverrides{" +
+                "host='" + host + '\'' +
+                ", port=" + port +
+                ", realm='" + realm + '\'' +
+                '}';
+    }
+}
diff --git a/address-space-controller/src/main/java/io/enmasse/controller/keycloak/KeycloakApi.java b/api-model/src/main/java/io/enmasse/user/api/RealmApi.java
similarity index 70%
rename from address-space-controller/src/main/java/io/enmasse/controller/keycloak/KeycloakApi.java
rename to api-model/src/main/java/io/enmasse/user/api/RealmApi.java
index 5720d1ca8..a78d6f871 100644
--- a/address-space-controller/src/main/java/io/enmasse/controller/keycloak/KeycloakApi.java
+++ b/api-model/src/main/java/io/enmasse/user/api/RealmApi.java
@@ -2,14 +2,14 @@
  * Copyright 2017-2018, EnMasse authors.
  * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html).
  */
-package io.enmasse.controller.keycloak;
+package io.enmasse.user.api;
 
 import io.enmasse.admin.model.v1.AuthenticationService;
 
 import java.util.Set;
 
-public interface KeycloakApi {
-    Set<String> getRealmNames(AuthenticationService authenticationService);
-    void createRealm(AuthenticationService authenticationService, String namespace, String realmName);
-    void deleteRealm(AuthenticationService authenticationService, String realmName);
+public interface RealmApi {
+    Set<String> getRealmNames(AuthenticationService authenticationService) throws Exception;
+    void createRealm(AuthenticationService authenticationService, String namespace, String realmName) throws Exception;
+    void deleteRealm(AuthenticationService authenticationService, String realmName) throws Exception;
 }
diff --git a/api-server/src/main/java/io/enmasse/api/server/ApiServer.java b/api-server/src/main/java/io/enmasse/api/server/ApiServer.java
index 69c2d0523..8ec4d142c 100644
--- a/api-server/src/main/java/io/enmasse/api/server/ApiServer.java
+++ b/api-server/src/main/java/io/enmasse/api/server/ApiServer.java
@@ -73,11 +73,13 @@ public class ApiServer extends AbstractVerticle {
 
         AuthenticationServiceRegistry authenticationServiceRegistry = new SchemaAuthenticationServiceRegistry(schemaProvider);
 
-        KeycloakFactory keycloakFactory = new KubeKeycloakFactory(client);
         Clock clock = Clock.systemUTC();
+        KeycloakFactory keycloakFactory = new KubeKeycloakFactory(client);
+        KeycloakUserApi keycloakUserApi = new KeycloakUserApi(keycloakFactory, clock, options.getUserApiTimeout());
+        schemaProvider.registerListener(newSchema -> keycloakUserApi.retainAuthenticationServices(newSchema.findAuthenticationServiceType(AuthenticationServiceType.standard)));
         UserApi userApi = new DelegateUserApi(Map.of(AuthenticationServiceType.none, new NullUserApi(),
                 AuthenticationServiceType.external, new NullUserApi(),
-                AuthenticationServiceType.standard, new KeycloakUserApi(keycloakFactory, clock, options.getUserApiTimeout())));
+                AuthenticationServiceType.standard, keycloakUserApi));
 
         String clientCa;
         String requestHeaderClientCa;
diff --git a/api-server/src/main/resources/swagger.json b/api-server/src/main/resources/swagger.json
index f9800513f..3d13df7dc 100644
--- a/api-server/src/main/resources/swagger.json
+++ b/api-server/src/main/resources/swagger.json
@@ -1433,11 +1433,25 @@
                 "authenticationService": {
                     "type": "object",
                     "properties": {
+                        "name": {
+                            "type": "string"
+                        },
                         "type": {
                             "type": "string"
                         },
-                        "details": {
-                            "type": "object"
+                        "overrides": {
+                            "type": "object",
+                            "properties": {
+                                "host": {
+                                    "type": "string"
+                                },
+                                "port": {
+                                    "type": "integer"
+                                },
+                                "realm": {
+                                    "type": "string"
+                                }
+                            }
                         }
                     }
                 },
diff --git a/documentation/common/restapi-reference.adoc b/documentation/common/restapi-reference.adoc
index ee443b7bc..7f61abd72 100644
--- a/documentation/common/restapi-reference.adoc
+++ b/documentation/common/restapi-reference.adoc
@@ -1990,12 +1990,28 @@ __required__|<<_io_enmasse_v1beta1_addressspacetype,io.enmasse.v1beta1.AddressSp
 [options="header", cols=".^3a,.^4a"]
 |===
 |Name|Schema
-|**details** +
-__optional__|object
+|**name** +
+__optional__|string
+|**overrides** +
+__optional__|<<_io_enmasse_v1beta1_addressspacespec_overrides,overrides>>
 |**type** +
 __optional__|string
 |===
 
+[[_io_enmasse_v1beta1_addressspacespec_overrides]]
+**overrides**
+
+[options="header", cols=".^3a,.^4a"]
+|===
+|Name|Schema
+|**host** +
+__optional__|string
+|**port** +
+__optional__|integer
+|**realm** +
+__optional__|string
+|===
+
 [[_io_enmasse_v1beta1_addressspacespec_endpoints]]
 **endpoints**
 
diff --git a/k8s-api/src/main/java/io/enmasse/k8s/api/CachingSchemaProvider.java b/k8s-api/src/main/java/io/enmasse/k8s/api/CachingSchemaProvider.java
index a92e89317..9d2b5b769 100644
--- a/k8s-api/src/main/java/io/enmasse/k8s/api/CachingSchemaProvider.java
+++ b/k8s-api/src/main/java/io/enmasse/k8s/api/CachingSchemaProvider.java
@@ -8,17 +8,23 @@ import io.enmasse.address.model.Schema;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.util.ArrayList;
 import java.util.List;
 
 public class CachingSchemaProvider implements SchemaProvider, Watcher<Schema> {
     private static final Logger log = LoggerFactory.getLogger(CachingSchemaProvider.class);
     private volatile Schema schema = null;
+    private final List<SchemaListener> listeners = new ArrayList<>();
 
     @Override
     public Schema getSchema() {
         return schema;
     }
 
+    public void registerListener(SchemaListener listener) {
+        listeners.add(listener);
+    }
+
     @Override
     public void onUpdate(List<Schema> items) {
         if (items == null || items.isEmpty()) {
@@ -26,5 +32,8 @@ public class CachingSchemaProvider implements SchemaProvider, Watcher<Schema> {
         }
         schema = items.get(0);
         log.info("Schema updated: {}", schema.printSchema());
+        for (SchemaListener listener : listeners) {
+            listener.onUpdate(schema);
+        }
     }
 }
diff --git a/k8s-api/src/main/java/io/enmasse/k8s/api/SchemaListener.java b/k8s-api/src/main/java/io/enmasse/k8s/api/SchemaListener.java
new file mode 100644
index 000000000..5f9f6fcf7
--- /dev/null
+++ b/k8s-api/src/main/java/io/enmasse/k8s/api/SchemaListener.java
@@ -0,0 +1,11 @@
+/*
+ * Copyright 2018, EnMasse authors.
+ * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html).
+ */
+package io.enmasse.k8s.api;
+
+import io.enmasse.address.model.Schema;
+
+public interface SchemaListener {
+    void onUpdate(Schema newSchema);
+}
diff --git a/keycloak-user-api/src/main/java/io/enmasse/user/keycloak/KeycloakUnavailableException.java b/keycloak-user-api/src/main/java/io/enmasse/user/keycloak/KeycloakUnavailableException.java
new file mode 100644
index 000000000..de9476119
--- /dev/null
+++ b/keycloak-user-api/src/main/java/io/enmasse/user/keycloak/KeycloakUnavailableException.java
@@ -0,0 +1,11 @@
+/*
+ * Copyright 2019, EnMasse authors.
+ * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html).
+ */
+package io.enmasse.user.keycloak;
+
+public class KeycloakUnavailableException extends RuntimeException {
+    public KeycloakUnavailableException(String message) {
+        super(message);
+    }
+}
diff --git a/keycloak-user-api/src/main/java/io/enmasse/user/keycloak/KeycloakUserApi.java b/keycloak-user-api/src/main/java/io/enmasse/user/keycloak/KeycloakUserApi.java
index 4a3bff35d..421b56764 100644
--- a/keycloak-user-api/src/main/java/io/enmasse/user/keycloak/KeycloakUserApi.java
+++ b/keycloak-user-api/src/main/java/io/enmasse/user/keycloak/KeycloakUserApi.java
@@ -8,6 +8,7 @@ package io.enmasse.user.keycloak;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import io.enmasse.admin.model.v1.AuthenticationService;
 import io.enmasse.k8s.util.TimeUtil;
+import io.enmasse.user.api.RealmApi;
 import io.enmasse.user.api.UserApi;
 import io.enmasse.user.model.v1.*;
 import io.fabric8.kubernetes.api.model.ObjectMetaBuilder;
@@ -38,7 +39,7 @@ import java.util.stream.Collectors;
 import static java.util.Optional.empty;
 
 
-public class KeycloakUserApi implements UserApi {
+public class KeycloakUserApi implements UserApi, RealmApi {
 
     private static final Logger log = LoggerFactory.getLogger(KeycloakUserApi.class);
 
@@ -47,16 +48,24 @@ public class KeycloakUserApi implements UserApi {
     private final Duration apiTimeout;
     private final Map<String, Keycloak> keycloakMap = new HashMap<>();
 
-    public KeycloakUserApi(KeycloakFactory keycloakFactory, Clock clock) {
-        this(keycloakFactory, clock, Duration.ZERO);
-    }
-
     public KeycloakUserApi(KeycloakFactory keycloakFactory, Clock clock, Duration apiTimeout) {
         this.keycloakFactory = keycloakFactory;
         this.clock = clock;
         this.apiTimeout = apiTimeout;
     }
 
+    public synchronized void retainAuthenticationServices(List<AuthenticationService> items) {
+        Set<String> desired = items.stream().map(a -> a.getMetadata().getName()).collect(Collectors.toSet());
+        Set<String> toRemove = new HashSet<>(keycloakMap.keySet());
+        toRemove.removeAll(desired);
+        for (String authService : toRemove) {
+            Keycloak keycloak = keycloakMap.remove(authService);
+            if (keycloak != null && !keycloak.isClosed()) {
+                keycloak.close();
+            }
+        }
+    }
+
     interface KeycloakHandler<T> {
         T handle(Keycloak keycloak) throws Exception;
     }
@@ -671,6 +680,36 @@ public class KeycloakUserApi implements UserApi {
 
     }
 
+    @Override
+    public Set<String> getRealmNames(AuthenticationService authenticationService) throws Exception {
+        return withKeycloak(authenticationService, kc -> kc.realms().findAll().stream()
+                .map(RealmRepresentation::getRealm)
+                .collect(Collectors.toSet()));
+    }
+
+    @Override
+    public void createRealm(AuthenticationService authenticationService, String namespace, String realmName) throws Exception {
+        final RealmRepresentation newRealm = new RealmRepresentation();
+        newRealm.setRealm(realmName);
+        newRealm.setEnabled(true);
+        newRealm.setPasswordPolicy("hashAlgorithm(scramsha1)");
+        newRealm.setAttributes(new HashMap<>());
+        newRealm.getAttributes().put("namespace", namespace);
+        newRealm.getAttributes().put("enmasse-realm", "true");
+
+        withKeycloak(authenticationService, kc -> {
+            kc.realms().create(newRealm);
+            return true;
+        });
+    }
+
+    @Override
+    public void deleteRealm(AuthenticationService authenticationService, String realmName) throws Exception {
+        withKeycloak(authenticationService, kc -> {
+            kc.realm(realmName).remove();
+            return true;
+        });
+    }
 
     public static String decodePart(final String part) {
         try {
diff --git a/keycloak-user-api/src/main/java/io/enmasse/user/keycloak/KubeKeycloakFactory.java b/keycloak-user-api/src/main/java/io/enmasse/user/keycloak/KubeKeycloakFactory.java
index ae10e2895..59332dbbe 100644
--- a/keycloak-user-api/src/main/java/io/enmasse/user/keycloak/KubeKeycloakFactory.java
+++ b/keycloak-user-api/src/main/java/io/enmasse/user/keycloak/KubeKeycloakFactory.java
@@ -48,7 +48,7 @@ public class KubeKeycloakFactory implements KeycloakFactory {
     @Override
     public Keycloak createInstance(AuthenticationService authenticationService) {
         if (!authenticationService.getSpec().getType().equals(AuthenticationServiceType.standard) || authenticationService.getStatus() == null) {
-            return null;
+            throw new KeycloakUnavailableException("Standard authentication service " + authenticationService.getMetadata().getName() + " is not available");
         }
         log.info("Using standard authentication service '{}'", authenticationService.getMetadata().getName());
 
diff --git a/pom.xml b/pom.xml
index 4917beb62..567701c9a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -559,6 +559,12 @@
                 <groupId>com.github.fge</groupId>
                 <artifactId>json-patch</artifactId>
                 <version>${json-patch.version}</version>
+                <exclusions>
+                    <exclusion>
+                        <groupId>com.google.code.findbugs</groupId>
+                        <artifactId>jsr305</artifactId>
+                    </exclusion>
+                </exclusions>
             </dependency>
 
             <!-- Test-only dependencies -->
diff --git a/service-broker/src/main/java/io/enmasse/osb/ServiceBroker.java b/service-broker/src/main/java/io/enmasse/osb/ServiceBroker.java
index 469312a07..8eefde1be 100644
--- a/service-broker/src/main/java/io/enmasse/osb/ServiceBroker.java
+++ b/service-broker/src/main/java/io/enmasse/osb/ServiceBroker.java
@@ -72,7 +72,7 @@ public class ServiceBroker extends AbstractVerticle {
         AddressSpaceApi addressSpaceApi = new ConfigMapAddressSpaceApi(client);
         AuthApi authApi = new KubeAuthApi(client, client.getConfiguration().getOauthToken());
 
-        UserApi userApi = createUserApi(options);
+        UserApi userApi = createUserApi(options, schemaProvider);
 
         ConsoleProxy consoleProxy = addressSpace -> {
             Route route = client.adapt(OpenShiftClient.class).routes().inNamespace(client.getNamespace()).withName(options.getConsoleProxyRouteName()).get();
@@ -106,11 +106,13 @@ public class ServiceBroker extends AbstractVerticle {
         }
     }
 
-    private UserApi createUserApi(ServiceBrokerOptions options) {
+    private UserApi createUserApi(ServiceBrokerOptions options, CachingSchemaProvider schemaProvider) {
         KeycloakFactory keycloakFactory = new KubeKeycloakFactory(client);
+        KeycloakUserApi keycloakUserApi = new KeycloakUserApi(keycloakFactory, Clock.systemUTC(), Duration.ZERO);
+        schemaProvider.registerListener(newSchema -> keycloakUserApi.retainAuthenticationServices(newSchema.findAuthenticationServiceType(AuthenticationServiceType.standard)));
         return new DelegateUserApi(Map.of(AuthenticationServiceType.none, new NullUserApi(),
                 AuthenticationServiceType.external, new NullUserApi(),
-                AuthenticationServiceType.standard, new KeycloakUserApi(keycloakFactory, Clock.systemUTC(), Duration.ZERO)));
+                AuthenticationServiceType.standard, keycloakUserApi));
     }
 
     private static void ensureRouteExists(OpenShiftClient client, ServiceBrokerOptions serviceBrokerOptions) throws IOException {
diff --git a/templates/address-space-controller/050-Deployment-address-space-controller.yaml b/templates/address-space-controller/050-Deployment-address-space-controller.yaml
index 7282b617e..eb756bdfe 100644
--- a/templates/address-space-controller/050-Deployment-address-space-controller.yaml
+++ b/templates/address-space-controller/050-Deployment-address-space-controller.yaml
@@ -7,6 +7,8 @@ metadata:
   name: address-space-controller
 spec:
   replicas: 1
+  strategy:
+    type: Recreate
   selector:
     matchLabels:
       app: enmasse
diff --git a/templates/csv/enmasse.clusterserviceversion.yaml b/templates/csv/enmasse.clusterserviceversion.yaml
index ce2d286d9..e32c43b81 100644
--- a/templates/csv/enmasse.clusterserviceversion.yaml
+++ b/templates/csv/enmasse.clusterserviceversion.yaml
@@ -572,6 +572,8 @@ spec:
       - name: address-space-controller
         spec:
           replicas: 1
+          strategy:
+            type: RollingUpdate
           selector:
             matchLabels:
               app: enmasse


Test patch:
diff --git a/address-space-controller/src/test/java/io/enmasse/controller/keycloak/RealmControllerTest.java b/address-space-controller/src/test/java/io/enmasse/controller/keycloak/RealmControllerTest.java
index 145694d5f..f8ee89db9 100644
--- a/address-space-controller/src/test/java/io/enmasse/controller/keycloak/RealmControllerTest.java
+++ b/address-space-controller/src/test/java/io/enmasse/controller/keycloak/RealmControllerTest.java
@@ -9,6 +9,7 @@ import io.enmasse.admin.model.v1.AuthenticationService;
 import io.enmasse.admin.model.v1.AuthenticationServiceBuilder;
 import io.enmasse.config.AnnotationKeys;
 import io.enmasse.k8s.api.AuthenticationServiceRegistry;
+import io.enmasse.user.api.RealmApi;
 import io.fabric8.kubernetes.api.model.ObjectMetaBuilder;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -50,7 +51,7 @@ public class RealmControllerTest {
         when(mockAuthenticationServiceRegistry.findAuthenticationServiceByType(eq(io.enmasse.admin.model.v1.AuthenticationServiceType.standard))).thenReturn(Arrays.asList(authenticationService));
         when(mockAuthenticationServiceRegistry.listAuthenticationServices()).thenReturn(Collections.singletonList(authenticationService));
 
-        manager = new RealmController(new KeycloakApi() {
+        manager = new RealmController(new RealmApi() {
             @Override
             public Set<String> getRealmNames(AuthenticationService auth) {
                 return new HashSet<>(realms);
diff --git a/api-model/src/test/java/io/enmasse/address/model/v1/address/SerializationTest.java b/api-model/src/test/java/io/enmasse/address/model/v1/address/SerializationTest.java
index a76640372..475469e49 100644
--- a/api-model/src/test/java/io/enmasse/address/model/v1/address/SerializationTest.java
+++ b/api-model/src/test/java/io/enmasse/address/model/v1/address/SerializationTest.java
@@ -608,9 +608,11 @@ public class SerializationTest {
                 "  \"plan\":\"myplan\"," +
                 "  \"authenticationService\": {" +
                 "     \"name\": \"external\"," +
-                "     \"host\": \"override.example.com\"," +
-                "     \"port\": 1234," +
-                "     \"realm\": \"override\"" +
+                "     \"overrides\": {" +
+                "         \"host\": \"override.example.com\"," +
+                "         \"port\": 1234," +
+                "         \"realm\": \"override\"" +
+                "     }" +
                 "  }" +
                 "}" +
                 "}";
@@ -618,9 +620,9 @@ public class SerializationTest {
         ObjectMapper mapper = new ObjectMapper();
         AddressSpace addressSpace = mapper.readValue(json, AddressSpace.class);
         assertThat(addressSpace.getSpec().getAuthenticationService().getName(), is("external"));
-        assertThat(addressSpace.getSpec().getAuthenticationService().getHost(), is("override.example.com"));
-        assertThat(addressSpace.getSpec().getAuthenticationService().getPort(), is(1234));
-        assertThat(addressSpace.getSpec().getAuthenticationService().getRealm(), is("override"));
+        assertThat(addressSpace.getSpec().getAuthenticationService().getOverrides().getHost(), is("override.example.com"));
+        assertThat(addressSpace.getSpec().getAuthenticationService().getOverrides().getPort(), is(1234));
+        assertThat(addressSpace.getSpec().getAuthenticationService().getOverrides().getRealm(), is("override"));
     }
 
     @Test
diff --git a/systemtests/README.md b/systemtests/README.md
index 91b090670..8dec76fd5 100644
--- a/systemtests/README.md
+++ b/systemtests/README.md
@@ -9,9 +9,11 @@ fallback to `kubectl`.
 It is assumed that you have EnMasse running already. If not, have a look at
 the [documentation](http://enmasse.io/documentation/master/openshift/#quickstart-messaging-gs).
 
-*NOTE*: On OKD, make sure `oc` is logged in to your cluster:
+*NOTE*: On OKD, make sure `oc` is logged in to your cluster with `cluster-admin` privileges. However you should not run the tests with `system:admin` user. You can give the privileges to a standard user :
 
-    oc login https://localhost:8443
+    oc login -u system:admin https://localhost:8443
+    oc adm policy add-cluster-role-to-user cluster-admin developer
+    oc login -u developer -p developer https://localhost:8443
 
 ## Running all tests
 
diff --git a/systemtests/src/main/java/io/enmasse/systemtest/ability/ExecutionListener.java b/systemtests/src/main/java/io/enmasse/systemtest/ability/ExecutionListener.java
index 5721b259e..2b9e730c4 100644
--- a/systemtests/src/main/java/io/enmasse/systemtest/ability/ExecutionListener.java
+++ b/systemtests/src/main/java/io/enmasse/systemtest/ability/ExecutionListener.java
@@ -24,7 +24,7 @@ import java.net.MalformedURLException;
 
 
 public class ExecutionListener implements TestExecutionListener {
-    static final Logger log = CustomLogger.getLogger();
+    private static final Logger log = CustomLogger.getLogger();
 
     @Override
     public void testPlanExecutionFinished(TestPlan testPlan) {
diff --git a/systemtests/src/main/java/io/enmasse/systemtest/ability/TestWatcher.java b/systemtests/src/main/java/io/enmasse/systemtest/ability/TestWatcher.java
new file mode 100644
index 000000000..b63931909
--- /dev/null
+++ b/systemtests/src/main/java/io/enmasse/systemtest/ability/TestWatcher.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright 2019, EnMasse authors.
+ * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html).
+ */
+package io.enmasse.systemtest.ability;
+
+import io.enmasse.systemtest.CustomLogger;
+import io.enmasse.systemtest.Environment;
+import io.enmasse.systemtest.Kubernetes;
+import io.enmasse.systemtest.cmdclients.KubeCMDClient;
+import org.junit.jupiter.api.extension.AfterTestExecutionCallback;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.slf4j.Logger;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+public class TestWatcher implements AfterTestExecutionCallback {
+    private static final Logger log = CustomLogger.getLogger();
+
+    @Override
+    public void afterTestExecution(ExtensionContext extensionContext) throws Exception {
+        Method testMethod = extensionContext.getRequiredTestMethod();
+        Class testClass = extensionContext.getRequiredTestClass();
+        if (extensionContext.getExecutionException().isPresent()) {
+            try {
+                log.info("Saving pod logs and info...");
+                Kubernetes kube = Kubernetes.getInstance();
+                Path path = Paths.get(
+                        Environment.getInstance().testLogDir(),
+                        "failed_test_logs",
+                        testClass.getName(),
+                        testMethod.getName());
+                Files.createDirectories(path);
+                kube.listPods().forEach(pod -> {
+                    kube.getContainersFromPod(pod.getMetadata().getName()).forEach(
+                            container -> {
+                                File filePath = new File(path.toString(), String.format("%s_%s.log", pod.getMetadata().getName(), container.getName()));
+                                try {
+                                    Files.write(Paths.get(filePath.toString()), kube.getLog(pod.getMetadata().getName(), container.getName()).getBytes());
+                                } catch (IOException e) {
+                                    log.warn("Cannot write file {}", filePath.getName());
+                                }
+                            }
+                    );
+                });
+                Files.write(Paths.get(path.toString(), "describe.txt"), KubeCMDClient.describePods(kube.getNamespace()).getStdOut().getBytes());
+                log.info("Pod logs and describe successfully stored into {}", path.toString());
+            } catch (Exception ex) {
+                log.warn("Cannot save pod logs and info: {}", ex.getMessage());
+            }
+        }
+    }
+}
diff --git a/systemtests/src/main/java/io/enmasse/systemtest/cmdclients/KubeCMDClient.java b/systemtests/src/main/java/io/enmasse/systemtest/cmdclients/KubeCMDClient.java
index 93055b33c..34670e1bb 100644
--- a/systemtests/src/main/java/io/enmasse/systemtest/cmdclients/KubeCMDClient.java
+++ b/systemtests/src/main/java/io/enmasse/systemtest/cmdclients/KubeCMDClient.java
@@ -276,7 +276,7 @@ public class KubeCMDClient extends CmdClient {
     }
 
     public static ExecutionResultData describePods(String namespace) {
-        return execute(DEFAULT_SYNC_TIMEOUT, true, "kubectl", "-n", namespace, "describe", "pods");
+        return execute(DEFAULT_SYNC_TIMEOUT, false, CMD, "-n", namespace, "describe", "pods");
     }
 
     public static ExecutionResultData createFromFile(String namespace, Path path) {
diff --git a/systemtests/src/main/java/io/enmasse/systemtest/utils/IoTUtils.java b/systemtests/src/main/java/io/enmasse/systemtest/utils/IoTUtils.java
index 66376d0df..941e7d89b 100644
--- a/systemtests/src/main/java/io/enmasse/systemtest/utils/IoTUtils.java
+++ b/systemtests/src/main/java/io/enmasse/systemtest/utils/IoTUtils.java
@@ -21,6 +21,7 @@ import static io.enmasse.systemtest.utils.AddressSpaceUtils.jsonToAdressSpace;
 
 import java.util.Arrays;
 import java.util.Map;
+import java.util.Objects;
 import java.util.concurrent.TimeUnit;
 
 public class IoTUtils {
@@ -52,17 +53,11 @@ public class IoTUtils {
         }
         if (!isReady) {
             String jsonStatus = config != null ? config.getStatus().getState() : "";
-            throw new IllegalStateException("IoTConfig " + config.getMetadata().getName() + " is not in Ready state within timeout: " + jsonStatus);
+            throw new IllegalStateException("IoTConfig " + Objects.requireNonNull(config).getMetadata().getName() + " is not in Ready state within timeout: " + jsonStatus);
         }
 
-        try {
-            TestUtils.waitUntilCondition("IoT Config to deploy", (phase)->allDeploymentsPresent(kubernetes), budget);
-            TestUtils.waitForNReplicas(kubernetes, EXPECTED_DEPLOYMENTS.length, IOT_LABELS, budget);
-        } catch (Exception e) {
-            TestUtils.streamNonReadyPods(kubernetes, config.getMetadata().getNamespace()).forEach(KubeCMDClient::dumpPodLogs);
-            KubeCMDClient.describePods(config.getMetadata().getNamespace());
-            throw e;
-        }
+        TestUtils.waitUntilCondition("IoT Config to deploy", (phase)->allDeploymentsPresent(kubernetes), budget);
+        TestUtils.waitForNReplicas(kubernetes, EXPECTED_DEPLOYMENTS.length, IOT_LABELS, budget);
     }
 
     private static boolean allDeploymentsPresent(Kubernetes kubernetes) {
diff --git a/systemtests/src/test/java/io/enmasse/systemtest/bases/TestBase.java b/systemtests/src/test/java/io/enmasse/systemtest/bases/TestBase.java
index 2abd016ca..80e3651f7 100644
--- a/systemtests/src/test/java/io/enmasse/systemtest/bases/TestBase.java
+++ b/systemtests/src/test/java/io/enmasse/systemtest/bases/TestBase.java
@@ -12,6 +12,7 @@ import io.enmasse.address.model.AddressSpaceSchemaList;
 import io.enmasse.systemtest.*;
 import io.enmasse.systemtest.ability.ITestBase;
 import io.enmasse.systemtest.ability.ITestSeparator;
+import io.enmasse.systemtest.ability.TestWatcher;
 import io.enmasse.systemtest.amqp.AmqpClient;
 import io.enmasse.systemtest.amqp.AmqpClientFactory;
 import io.enmasse.systemtest.amqp.ReceiverStatus;
@@ -44,6 +45,7 @@ import org.hamcrest.Matchers;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.TestInstance;
+import org.junit.jupiter.api.extension.ExtendWith;
 import org.slf4j.Logger;
 
 import javax.jms.DeliveryMode;
@@ -72,10 +74,11 @@ import static org.junit.jupiter.api.Assertions.*;
 /**
  * Base class for all tests
  */
+@ExtendWith(TestWatcher.class)
 @TestInstance(TestInstance.Lifecycle.PER_CLASS)
 public abstract class TestBase implements ITestBase, ITestSeparator {
     protected static final Environment environment = Environment.getInstance();
-    protected static final UserCredentials clusterUser =  new UserCredentials(KubeCMDClient.getOCUser());
+    protected static final UserCredentials clusterUser = new UserCredentials(KubeCMDClient.getOCUser());
     protected static final Kubernetes kubernetes = Kubernetes.getInstance();
     protected static final GlobalLogCollector logCollector = new GlobalLogCollector(kubernetes,
             new File(environment.testLogDir()));