Repository: quarkusio/quarkus
Issue: ConcurrentModificationException in TransactionEntityManagers.getEntityManager

**Describe the bug**

I get `ConcurrentModificationException` at the start of a `ab` (https://httpd.apache.org/docs/2.4/programs/ab.html) benchmark session if the benchmarked REST endpoint needs database access. The issue only seems to appear if a concurrency level > 4 (i.e. `ab -c 5`) is used for the benchmark and `ab` is started prior to any request involving database access.

I think the concurrency value may depend on your machine - I am testing this on an Intel Core i7-6700HQ (4 cores, 8 threads) and with 5 I am able to trigger the bug ~90% of the time.

I wrote a little application displaying the issue. It uses:

* `quarkus-hibernate-orm-panache`
* `quarkus-jdbc-h2`
* `quarkus-resteasy-jsonb`

It can be found at https://github.com/j-be/quarkus-concurrent-modification-bug

I did NOT yet check native images yet, just the good-old ` java -jar`. I will update this report if I find out anything on that front.

I am working on a pull-request and will publish it shortly.

**Expected behavior**
No `ConcurrentModificationException` when running 

**Actual behavior**
```
2019-06-16 14:29:19,306 ERROR [io.und.request] (executor-thread-3) UT005023: Exception handling request to /genre: org.jboss.resteasy.spi.UnhandledException: java.util.ConcurrentModificationException
	at org.jboss.resteasy.core.ExceptionHandler.handleApplicationException(ExceptionHandler.java:106)
	at org.jboss.resteasy.core.ExceptionHandler.handleException(ExceptionHandler.java:372)
	at org.jboss.resteasy.core.SynchronousDispatcher.writeException(SynchronousDispatcher.java:209)
	at org.jboss.resteasy.core.SynchronousDispatcher.invoke(SynchronousDispatcher.java:496)
	at org.jboss.resteasy.core.SynchronousDispatcher.lambda$invoke$4(SynchronousDispatcher.java:252)
	at org.jboss.resteasy.core.SynchronousDispatcher.lambda$preprocess$0(SynchronousDispatcher.java:153)
	at org.jboss.resteasy.core.interception.jaxrs.PreMatchContainerRequestContext.filter(PreMatchContainerRequestContext.java:362)
	at org.jboss.resteasy.core.SynchronousDispatcher.preprocess(SynchronousDispatcher.java:156)
	at org.jboss.resteasy.core.SynchronousDispatcher.invoke(SynchronousDispatcher.java:238)
	at org.jboss.resteasy.plugins.server.servlet.ServletContainerDispatcher.service(ServletContainerDispatcher.java:234)
	at io.quarkus.resteasy.runtime.ResteasyFilter$ResteasyResponseWrapper.sendError(ResteasyFilter.java:72)
	at io.undertow.servlet.handlers.DefaultServlet.doGet(DefaultServlet.java:175)
	at javax.servlet.http.HttpServlet.service(HttpServlet.java:686)
	at javax.servlet.http.HttpServlet.service(HttpServlet.java:791)
	at io.undertow.servlet.handlers.ServletHandler.handleRequest(ServletHandler.java:74)
	at io.undertow.servlet.handlers.FilterHandler$FilterChainImpl.doFilter(FilterHandler.java:129)
	at io.quarkus.resteasy.runtime.ResteasyFilter.doFilter(ResteasyFilter.java:43)
	at io.undertow.servlet.core.ManagedFilter.doFilter(ManagedFilter.java:61)
	at io.undertow.servlet.handlers.FilterHandler$FilterChainImpl.doFilter(FilterHandler.java:131)
	at io.undertow.servlet.handlers.FilterHandler.handleRequest(FilterHandler.java:84)
	at io.undertow.servlet.handlers.security.ServletSecurityRoleHandler.handleRequest(ServletSecurityRoleHandler.java:62)
	at io.undertow.servlet.handlers.ServletChain$1.handleRequest(ServletChain.java:68)
	at io.undertow.servlet.handlers.ServletDispatchingHandler.handleRequest(ServletDispatchingHandler.java:36)
	at io.undertow.servlet.handlers.security.SSLInformationAssociationHandler.handleRequest(SSLInformationAssociationHandler.java:132)
	at io.undertow.servlet.handlers.security.ServletAuthenticationCallHandler.handleRequest(ServletAuthenticationCallHandler.java:57)
	at io.undertow.server.handlers.PredicateHandler.handleRequest(PredicateHandler.java:43)
	at io.undertow.security.handlers.AbstractConfidentialityHandler.handleRequest(AbstractConfidentialityHandler.java:46)
	at io.undertow.servlet.handlers.security.ServletConfidentialityConstraintHandler.handleRequest(ServletConfidentialityConstraintHandler.java:64)
	at io.undertow.security.handlers.AuthenticationMechanismsHandler.handleRequest(AuthenticationMechanismsHandler.java:60)
	at io.undertow.servlet.handlers.security.CachedAuthenticatedSessionHandler.handleRequest(CachedAuthenticatedSessionHandler.java:77)
	at io.undertow.security.handlers.AbstractSecurityContextAssociationHandler.handleRequest(AbstractSecurityContextAssociationHandler.java:43)
	at io.undertow.server.handlers.PredicateHandler.handleRequest(PredicateHandler.java:43)
	at io.undertow.server.handlers.PredicateHandler.handleRequest(PredicateHandler.java:43)
	at io.undertow.servlet.handlers.ServletInitialHandler.handleFirstRequest(ServletInitialHandler.java:292)
	at io.undertow.servlet.handlers.ServletInitialHandler.access$100(ServletInitialHandler.java:81)
	at io.undertow.servlet.handlers.ServletInitialHandler$2.call(ServletInitialHandler.java:138)
	at io.undertow.servlet.handlers.ServletInitialHandler$2.call(ServletInitialHandler.java:135)
	at io.undertow.servlet.core.ServletRequestContextThreadSetupAction$1.call(ServletRequestContextThreadSetupAction.java:48)
	at io.undertow.servlet.core.ContextClassLoaderSetupAction$1.call(ContextClassLoaderSetupAction.java:43)
	at io.quarkus.undertow.runtime.UndertowDeploymentTemplate$7$1$1.call(UndertowDeploymentTemplate.java:469)
	at io.undertow.servlet.handlers.ServletInitialHandler.dispatchRequest(ServletInitialHandler.java:272)
	at io.undertow.servlet.handlers.ServletInitialHandler.handleRequest(ServletInitialHandler.java:197)
	at io.undertow.server.handlers.HttpContinueReadHandler.handleRequest(HttpContinueReadHandler.java:65)
	at io.quarkus.undertow.runtime.UndertowDeploymentTemplate$1.handleRequest(UndertowDeploymentTemplate.java:104)
	at io.undertow.server.handlers.CanonicalPathHandler.handleRequest(CanonicalPathHandler.java:49)
	at io.quarkus.undertow.deployment.devmode.UndertowHotReplacementSetup.handleHotDeploymentRequest(UndertowHotReplacementSetup.java:77)
	at io.quarkus.undertow.deployment.devmode.UndertowHotReplacementSetup$1$1.handleRequest(UndertowHotReplacementSetup.java:56)
	at io.undertow.server.Connectors.executeRootHandler(Connectors.java:364)
	at io.undertow.server.HttpServerExchange$1.run(HttpServerExchange.java:830)
	at io.quarkus.runtime.CleanableExecutor$CleaningRunnable.run(CleanableExecutor.java:243)
	at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
	at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
	at org.jboss.threads.ContextClassLoaderSavingRunnable.run(ContextClassLoaderSavingRunnable.java:35)
	at org.jboss.threads.EnhancedQueueExecutor.safeRun(EnhancedQueueExecutor.java:2011)
	at org.jboss.threads.EnhancedQueueExecutor$ThreadBody.doRunTask(EnhancedQueueExecutor.java:1538)
	at org.jboss.threads.EnhancedQueueExecutor$ThreadBody.run(EnhancedQueueExecutor.java:1395)
	at org.jboss.threads.DelegatingRunnable.run(DelegatingRunnable.java:29)
	at org.jboss.threads.ThreadLocalResettingRunnable.run(ThreadLocalResettingRunnable.java:32)
	at java.base/java.lang.Thread.run(Thread.java:834)
	at org.jboss.threads.JBossThread.run(JBossThread.java:479)
Caused by: java.util.ConcurrentModificationException
	at java.base/java.util.HashMap.computeIfAbsent(HashMap.java:1134)
	at io.quarkus.hibernate.orm.runtime.TransactionEntityManagers.getEntityManager(TransactionEntityManagers.java:51)
	at io.quarkus.hibernate.orm.runtime.JPAResourceReferenceProvider$1.delegate(JPAResourceReferenceProvider.java:54)
	at io.quarkus.hibernate.orm.runtime.entitymanager.ForwardingEntityManager.createQuery(ForwardingEntityManager.java:158)
	at io.quarkus.hibernate.orm.panache.runtime.JpaOperations.count(JpaOperations.java:293)
	at org.duckdns.owly.quarkus.service.GerneService.count(GerneService.java)
	at org.duckdns.owly.quarkus.service.GerneService_ClientProxy.count(Unknown Source)
	at org.duckdns.owly.quarkus.resources.GenreResource.getCount(GenreResource.java:23)
	at org.duckdns.owly.quarkus.resources.GenreResource_ClientProxy.getCount(Unknown Source)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at org.jboss.resteasy.core.MethodInjectorImpl.invoke(MethodInjectorImpl.java:151)
	at org.jboss.resteasy.core.MethodInjectorImpl.lambda$invoke$3(MethodInjectorImpl.java:122)
	at java.base/java.util.concurrent.CompletableFuture.uniApplyNow(CompletableFuture.java:680)
	at java.base/java.util.concurrent.CompletableFuture.uniApplyStage(CompletableFuture.java:658)
	at java.base/java.util.concurrent.CompletableFuture.thenApply(CompletableFuture.java:2094)
	at java.base/java.util.concurrent.CompletableFuture.thenApply(CompletableFuture.java:143)
	at org.jboss.resteasy.core.MethodInjectorImpl.invoke(MethodInjectorImpl.java:122)
	at org.jboss.resteasy.core.ResourceMethodInvoker.internalInvokeOnTarget(ResourceMethodInvoker.java:568)
	at org.jboss.resteasy.core.ResourceMethodInvoker.invokeOnTargetAfterFilter(ResourceMethodInvoker.java:442)
	at org.jboss.resteasy.core.ResourceMethodInvoker.lambda$invokeOnTarget$2(ResourceMethodInvoker.java:396)
	at org.jboss.resteasy.core.interception.jaxrs.PreMatchContainerRequestContext.filter(PreMatchContainerRequestContext.java:362)
	at org.jboss.resteasy.core.ResourceMethodInvoker.invokeOnTarget(ResourceMethodInvoker.java:398)
	at org.jboss.resteasy.core.ResourceMethodInvoker.invoke(ResourceMethodInvoker.java:367)
	at org.jboss.resteasy.core.ResourceMethodInvoker.lambda$invoke$1(ResourceMethodInvoker.java:341)
	at java.base/java.util.concurrent.CompletableFuture.uniComposeStage(CompletableFuture.java:1106)
	at java.base/java.util.concurrent.CompletableFuture.thenCompose(CompletableFuture.java:2235)
	at java.base/java.util.concurrent.CompletableFuture.thenCompose(CompletableFuture.java:143)
	at org.jboss.resteasy.core.ResourceMethodInvoker.invoke(ResourceMethodInvoker.java:341)
	at org.jboss.resteasy.core.SynchronousDispatcher.invoke(SynchronousDispatcher.java:477)
	... 56 more
```

**To Reproduce**
Prerequisits:
* Ubuntu Linux 18.04 LTS with:
  * `mvn 3.6.0` from package repositories
  *  `openJDK 11` from package repositories
  * `apache2-utils` from package repositories

Steps to reproduce the behavior:
1. clone https://github.com/j-be/quarkus-concurrent-modification-bug
2. Run `mvn clean package quarkus:dev` (you may `-DskipTests` if you want)
3. Open a second terminal and run `ab -n 100 -c 5 localhost:8080/genre`
4. You should now see a `java.util.ConcurrentModificationException`. If not, stop the quarkus server, start it again and increase `ab`'s concurrency.

**Configuration**
I doubt this is of any relevance, yet for completeness sake: 
```properties
# Configuration file
# key = value

# configure your datasource
quarkus.datasource.url = jdbc:h2:mem:test;DB_CLOSE_DELAY=-1;DB_CLOSE_ON_EXIT=TRUE
quarkus.datasource.driver = org.h2.Driver
quarkus.datasource.username = sa
quarkus.datasource.password =

# drop and create the database at startup (use `update` to only update the schema)
quarkus.hibernate-orm.database.generation = create
```

**Screenshots**
Not applicable.

**Environment (please complete the following information):**
 - Output of `uname -a`:
Linux albatros 4.15.0-51-generic #55-Ubuntu SMP Wed May 15 14:27:21 UTC 2019 x86_64 x86_64 x86_64 GNU/Linux 
 - Output of `java -version`: 
openjdk version "11.0.3" 2019-04-16
OpenJDK Runtime Environment (build 11.0.3+7-Ubuntu-1ubuntu218.04.1)
OpenJDK 64-Bit Server VM (build 11.0.3+7-Ubuntu-1ubuntu218.04.1, mixed mode, sharing)
 - GraalVM version (if different from Java): 
Not applicable (yet)
 - Quarkus version or git rev: 
both `0.16.1` from Maven Central and current HEAD of master (59c85e3ba31535be5ef9bfe1566304d3ae31bb1a)

**Additional context**
Not applicable (I guess)

Changed files:
extensions/hibernate-orm/runtime/src/main/java/io/quarkus/hibernate/orm/runtime/TransactionEntityManagers.java

diff --git a/extensions/hibernate-orm/runtime/src/main/java/io/quarkus/hibernate/orm/runtime/TransactionEntityManagers.java b/extensions/hibernate-orm/runtime/src/main/java/io/quarkus/hibernate/orm/runtime/TransactionEntityManagers.java
index 056d4c63a6a..69407b17021 100644
--- a/extensions/hibernate-orm/runtime/src/main/java/io/quarkus/hibernate/orm/runtime/TransactionEntityManagers.java
+++ b/extensions/hibernate-orm/runtime/src/main/java/io/quarkus/hibernate/orm/runtime/TransactionEntityManagers.java
@@ -1,8 +1,9 @@
 package io.quarkus.hibernate.orm.runtime;
 
-import java.util.HashMap;
-import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
 
+import javax.enterprise.context.ApplicationScoped;
 import javax.enterprise.inject.Instance;
 import javax.inject.Inject;
 import javax.persistence.EntityManager;
@@ -11,13 +12,14 @@
 
 import io.quarkus.hibernate.orm.runtime.entitymanager.TransactionScopedEntityManager;
 
+@ApplicationScoped
 public class TransactionEntityManagers {
 
     @Inject
-    TransactionSynchronizationRegistry tsr;
+    TransactionSynchronizationRegistry transactionSynchronizationRegistry;
 
     @Inject
-    TransactionManager tm;
+    TransactionManager transactionManager;
 
     @Inject
     JPAConfig jpaConfig;
@@ -25,16 +27,20 @@ public class TransactionEntityManagers {
     @Inject
     Instance<RequestScopedEntityManagerHolder> requestScopedEntityManagers;
 
-    private final Map<String, TransactionScopedEntityManager> managers;
+    private final ConcurrentMap<String, TransactionScopedEntityManager> managers;
 
     public TransactionEntityManagers() {
-        this.managers = new HashMap<>();
+        this.managers = new ConcurrentHashMap<>();
     }
 
     public EntityManager getEntityManager(String unitName) {
-        return managers.computeIfAbsent(unitName,
-                un -> new TransactionScopedEntityManager(tm, tsr, jpaConfig.getEntityManagerFactory(un), unitName,
-                        requestScopedEntityManagers));
+        TransactionScopedEntityManager entityManager = managers.get(unitName);
+        if (entityManager != null) {
+            return entityManager;
+        }
+        return managers.computeIfAbsent(unitName, (un) -> new TransactionScopedEntityManager(
+                transactionManager, transactionSynchronizationRegistry, jpaConfig.getEntityManagerFactory(un), un,
+                requestScopedEntityManagers));
     }
 
 }