Project: usergrid
Issue USERGRID-442: Investigate ElasticSearch timeout such that a long-running GC does not pause all ES and Tomcat
I have seen as many as 16 tomcat nodes pausing at the same time across the cluster. I took a stack trace and have attached it. This one jumped out to me: "http-bio-8080-exec-2" daemon prio=10 tid=0x00007f0cbc003800 nid=0x20ff waiting on condition [0x00007f0d370dd000] java.lang.Thread.State: WAITING (parking) sun.misc.Unsafe.park(Native Method)	- parking to wait for <0x00000007ee1de4d8> (a org.elasticsearch.common.util.concurrent.BaseFuture$Sync) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:834) java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:994) java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1303) org.elasticsearch.common.util.concurrent.BaseFuture$Sync.get(BaseFuture.java:274) org.elasticsearch.common.util.concurrent.BaseFuture.get(BaseFuture.java:113) org.elasticsearch.action.support.AdapterActionFuture.actionGet(AdapterActionFuture.java:45) org.apache.usergrid.persistence.index.impl.EsEntityIndexImpl.search(EsEntityIndexImpl.java:357) org.apache.usergrid.corepersistence.CpRelationManager.searchCollection(CpRelationManager.java:963) org.apache.usergrid.corepersistence.CpEntityManager.searchCollection(CpEntityManager.java:624) org.apache.usergrid.corepersistence.CpEntityManagerFactory.lookupApplication(CpEntityManagerFactory.java:433) org.apache.usergrid.rest.organizations.OrganizationResource.getApplicationByName(OrganizationResource.java:137) sun.reflect.GeneratedMethodAccessor113.invoke(Unknown Source) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:601) com.sun.jersey.server.impl.uri.rules.SubLocatorRule.dispatch(SubLocatorRule.java:197) com.sun.jersey.server.impl.uri.rules.SubLocatorRule.invokeSubLocator(SubLocatorRule.java:183) com.sun.jersey.server.impl.uri.rules.SubLocatorRule.accept(SubLocatorRule.java:110) com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147) com.sun.jersey.server.impl.uri.rules.SubLocatorRule.accept(SubLocatorRule.java:137) com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147) com.sun.jersey.server.impl.uri.rules.ResourceClassRule.accept(ResourceClassRule.java:108) com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147) com.sun.jersey.server.impl.uri.rules.RootResourceClassesRule.accept(RootResourceClassesRule.java:84) com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1542) com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1473) com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1419) com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1409) com.sun.jersey.spi.container.servlet.WebComponent.service(WebComponent.java:409) com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:540) com.sun.jersey.spi.container.servlet.ServletContainer.doFilter(ServletContainer.java:909) com.sun.jersey.spi.container.servlet.ServletContainer.doFilter(ServletContainer.java:857) com.sun.jersey.spi.container.servlet.ServletContainer.doFilter(ServletContainer.java:811) org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241) org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208) org.apache.shiro.web.servlet.AbstractShiroFilter.executeChain(AbstractShiroFilter.java:449) org.apache.shiro.web.servlet.AbstractShiroFilter$1.call(AbstractShiroFilter.java:365) org.apache.shiro.subject.support.SubjectCallable.doCall(SubjectCallable.java:90) org.apache.shiro.subject.support.SubjectCallable.call(SubjectCallable.java:83) org.apache.shiro.subject.support.DelegatingSubject.execute(DelegatingSubject.java:383) org.apache.shiro.web.servlet.AbstractShiroFilter.doFilterInternal(AbstractShiroFilter.java:362) org.apache.shiro.web.servlet.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:125) org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:346) org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:259) org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241) org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208) org.apache.usergrid.rest.filters.ContentTypeFilter.doFilter(ContentTypeFilter.java:92) org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241) org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208) org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:220) org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:122) org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:503) org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:170) org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:103) org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:950) org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:116) org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:421) org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1070) org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:611) org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:314)	- locked <0x00000007eda23278> (a org.apache.tomcat.util.net.SocketWrapper) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61) java.lang.Thread.run(Thread.java:722) and Todd called out this one: "collectiontasks-18" daemon prio=10 tid=0x00007f0cfc004000 nid=0x21b7 waiting on condition [0x00007f0cb87c6000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007ed32ccf0> (a org.elasticsearch.common.util.concurrent.BaseFuture$Sync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:834) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:994) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1303) at org.elasticsearch.common.util.concurrent.BaseFuture$Sync.get(BaseFuture.java:274) at org.elasticsearch.common.util.concurrent.BaseFuture.get(BaseFuture.java:113) at org.elasticsearch.action.support.AdapterActionFuture.actionGet(AdapterActionFuture.java:45) at org.apache.usergrid.persistence.index.impl.EsEntityIndexImpl.deletePreviousVersions(EsEntityIndexImpl.java:533) at org.apache.usergrid.corepersistence.events.EntityVersionCreatedHandler.versionCreated(EntityVersionCreatedHandler.java:67) at org.apache.usergrid.persistence.collection.impl.EntityVersionCreatedTask.fireEvents(EntityVersionCreatedTask.java:97) at org.apache.usergrid.persistence.collection.impl.EntityVersionCreatedTask.call(EntityVersionCreatedTask.java:83) at org.apache.usergrid.persistence.collection.impl.EntityVersionCreatedTask.call(EntityVersionCreatedTask.java:38) at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334) at java.util.concurrent.FutureTask.run(FutureTask.java:166) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:722)
Story points: 3