Project: usergrid
Issue USERGRID-506: Numerous Null Pointer Exceptions on bulk deletes
I am doing some bulk deletes and getting many NPE's and errors in the log files. During this time, Tomcat or ELB seems enticed to return many 503's. If this goes on too long the tomcats become unresponsive. Got some stack traces... 2015-03-24 20:01:55,925 [http-bio-8080-exec-5] ERROR org.apache.usergrid.corepersistence.CpEntityManager- Unable to load entity saparticl:03b6b5e4-c906-11e4-821c-651cb62f6d39 java.lang.NullPointerException org.apache.usergrid.corepersistence.CpEntityManager.validate(CpEntityManager.java:956) org.apache.usergrid.corepersistence.CpEntityManager.validate(CpEntityManager.java:942) org.apache.usergrid.corepersistence.CpEntityManager.getDictionaryAsMap(CpEntityManager.java:1205) org.apache.usergrid.corepersistence.CpEntityManager.getDictionaryAsSet(CpEntityManager.java:1119) org.apache.usergrid.services.AbstractService.getConnectedTypesSet(AbstractService.java:487) org.apache.usergrid.services.AbstractService.importEntity(AbstractService.java:355) org.apache.usergrid.services.AbstractService.importEntities(AbstractService.java:402) org.apache.usergrid.services.AbstractService.importEntities(AbstractService.java:414) org.apache.usergrid.services.AbstractCollectionService.deleteItemsByQuery(AbstractCollectionService.java:528) org.apache.usergrid.services.AbstractService.invokeItemsWithQuery(AbstractService.java:703) org.apache.usergrid.services.AbstractService.invoke(AbstractService.java:625) org.apache.usergrid.services.AbstractService.invoke(AbstractService.java:544) org.apache.usergrid.services.ServiceRequest.execute(ServiceRequest.java:226) org.apache.usergrid.services.ServiceRequest.execute(ServiceRequest.java:193) org.apache.usergrid.rest.applications.ServiceResource.executeServiceRequest(ServiceResource.java:251) org.apache.usergrid.rest.applications.ServiceResource.executeDelete(ServiceResource.java:437) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:601) com.sun.jersey.spi.container.JavaMethodInvokerFactory$1.invoke(JavaMethodInvokerFactory.java:60) com.sun.jersey.server.impl.model.method.dispatch.AbstractResourceMethodDispatchProvider$TypeOutInvoker._dispatch(AbstractResourceMethodDispatchProvider.java:185) com.sun.jersey.server.impl.model.method.dispatch.ResourceJavaMethodDispatcher.dispatch(ResourceJavaMethodDispatcher.java:75) com.sun.jersey.server.impl.uri.rules.HttpMethodRule.accept(HttpMethodRule.java:302) com.sun.jersey.server.impl.uri.rules.SubLocatorRule.accept(SubLocatorRule.java:137) com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147) com.sun.jersey.server.impl.uri.rules.SubLocatorRule.accept(SubLocatorRule.java:137) com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147) com.sun.jersey.server.impl.uri.rules.SubLocatorRule.accept(SubLocatorRule.java:137) com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147) com.sun.jersey.server.impl.uri.rules.ResourceClassRule.accept(ResourceClassRule.java:108) com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147) com.sun.jersey.server.impl.uri.rules.RootResourceClassesRule.accept(RootResourceClassesRule.java:84) com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1542) com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1473) com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1419) com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1409) com.sun.jersey.spi.container.servlet.WebComponent.service(WebComponent.java:409) com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:540) com.sun.jersey.spi.container.servlet.ServletContainer.doFilter(ServletContainer.java:909) com.sun.jersey.spi.container.servlet.ServletContainer.doFilter(ServletContainer.java:857) com.sun.jersey.spi.container.servlet.ServletContainer.doFilter(ServletContainer.java:811) org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241) org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208) org.apache.shiro.web.servlet.AbstractShiroFilter.executeChain(AbstractShiroFilter.java:449) org.apache.shiro.web.servlet.AbstractShiroFilter$1.call(AbstractShiroFilter.java:365) org.apache.shiro.subject.support.SubjectCallable.doCall(SubjectCallable.java:90) org.apache.shiro.subject.support.SubjectCallable.call(SubjectCallable.java:83) org.apache.shiro.subject.support.DelegatingSubject.execute(DelegatingSubject.java:383) org.apache.shiro.web.servlet.AbstractShiroFilter.doFilterInternal(AbstractShiroFilter.java:362) org.apache.shiro.web.servlet.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:125) org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:346) org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:259) org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241) org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208) org.apache.usergrid.rest.filters.ContentTypeFilter.doFilter(ContentTypeFilter.java:92) org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241) org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208) org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:220) org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:122) org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:503) org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:170) org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:103) org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:950) org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:116) org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:421) org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1070) org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:611) org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:314) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61) java.lang.Thread.run(Thread.java:722) 2015-03-24 20:01:56,030 [http-bio-8080-exec-5] ERROR org.apache.usergrid.corepersistence.CpEntityManager- Unable to load entity saparticl:03d8bcca-c906-11e4-bd5d-bdad5573d5fc java.lang.NullPointerException org.apache.usergrid.corepersistence.CpEntityManager.validate(CpEntityManager.java:956) org.apache.usergrid.corepersistence.CpEntityManager.validate(CpEntityManager.java:942) org.apache.usergrid.corepersistence.CpEntityManager.getDictionaryAsMap(CpEntityManager.java:1205) org.apache.usergrid.corepersistence.CpEntityManager.getDictionaryAsSet(CpEntityManager.java:1119) org.apache.usergrid.services.AbstractService.getConnectedTypesSet(AbstractService.java:487) org.apache.usergrid.services.AbstractService.importEntity(AbstractService.java:355) org.apache.usergrid.services.AbstractService.importEntities(AbstractService.java:402) org.apache.usergrid.services.AbstractService.importEntities(AbstractService.java:414) org.apache.usergrid.services.AbstractCollectionService.deleteItemsByQuery(AbstractCollectionService.java:528) org.apache.usergrid.services.AbstractService.invokeItemsWithQuery(AbstractService.java:703) org.apache.usergrid.services.AbstractService.invoke(AbstractService.java:625) org.apache.usergrid.services.AbstractService.invoke(AbstractService.java:544) org.apache.usergrid.services.ServiceRequest.execute(ServiceRequest.java:226) org.apache.usergrid.services.ServiceRequest.execute(ServiceRequest.java:193) org.apache.usergrid.rest.applications.ServiceResource.executeServiceRequest(ServiceResource.java:251) org.apache.usergrid.rest.applications.ServiceResource.executeDelete(ServiceResource.java:437) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:601) com.sun.jersey.spi.container.JavaMethodInvokerFactory$1.invoke(JavaMethodInvokerFactory.java:60) com.sun.jersey.server.impl.model.method.dispatch.AbstractResourceMethodDispatchProvider$TypeOutInvoker._dispatch(AbstractResourceMethodDispatchProvider.java:185) com.sun.jersey.server.impl.model.method.dispatch.ResourceJavaMethodDispatcher.dispatch(ResourceJavaMethodDispatcher.java:75) com.sun.jersey.server.impl.uri.rules.HttpMethodRule.accept(HttpMethodRule.java:302) com.sun.jersey.server.impl.uri.rules.SubLocatorRule.accept(SubLocatorRule.java:137) com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147) com.sun.jersey.server.impl.uri.rules.SubLocatorRule.accept(SubLocatorRule.java:137) com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147) com.sun.jersey.server.impl.uri.rules.SubLocatorRule.accept(SubLocatorRule.java:137) com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147) com.sun.jersey.server.impl.uri.rules.ResourceClassRule.accept(ResourceClassRule.java:108) com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147) com.sun.jersey.server.impl.uri.rules.RootResourceClassesRule.accept(RootResourceClassesRule.java:84) com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1542) com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1473) com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1419) com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1409) com.sun.jersey.spi.container.servlet.WebComponent.service(WebComponent.java:409) com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:540) com.sun.jersey.spi.container.servlet.ServletContainer.doFilter(ServletContainer.java:909) com.sun.jersey.spi.container.servlet.ServletContainer.doFilter(ServletContainer.java:857) com.sun.jersey.spi.container.servlet.ServletContainer.doFilter(ServletContainer.java:811) org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241) org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208) org.apache.shiro.web.servlet.AbstractShiroFilter.executeChain(AbstractShiroFilter.java:449) org.apache.shiro.web.servlet.AbstractShiroFilter$1.call(AbstractShiroFilter.java:365) org.apache.shiro.subject.support.SubjectCallable.doCall(SubjectCallable.java:90) org.apache.shiro.subject.support.SubjectCallable.call(SubjectCallable.java:83) org.apache.shiro.subject.support.DelegatingSubject.execute(DelegatingSubject.java:383) org.apache.shiro.web.servlet.AbstractShiroFilter.doFilterInternal(AbstractShiroFilter.java:362) org.apache.shiro.web.servlet.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:125) org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:346) org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:259) org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241) org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208) org.apache.usergrid.rest.filters.ContentTypeFilter.doFilter(ContentTypeFilter.java:92) org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241) org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208) org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:220) org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:122) org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:503) org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:170) org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:103) org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:950) org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:116) org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:421) org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1070) org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:611) org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:314) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61) java.lang.Thread.run(Thread.java:722) 2015-03-24 20:01:56,033 [http-bio-8080-exec-5] ERROR org.apache.usergrid.corepersistence.CpEntityManager- Unable to load entity saparticl:03d8bcca-c906-11e4-bd5d-bdad5573d5fc java.lang.NullPointerException org.apache.usergrid.corepersistence.CpEntityManager.validate(CpEntityManager.java:956) org.apache.usergrid.corepersistence.CpEntityManager.validate(CpEntityManager.java:942) org.apache.usergrid.corepersistence.CpEntityManager.getDictionaryAsMap(CpEntityManager.java:1205) org.apache.usergrid.corepersistence.CpEntityManager.getDictionaryAsSet(CpEntityManager.java:1119) org.apache.usergrid.services.AbstractService.getConnectingTypesSet(AbstractService.java:505) org.apache.usergrid.services.AbstractService.importEntity(AbstractService.java:364) org.apache.usergrid.services.AbstractService.importEntities(AbstractService.java:402) org.apache.usergrid.services.AbstractService.importEntities(AbstractService.java:414) org.apache.usergrid.services.AbstractCollectionService.deleteItemsByQuery(AbstractCollectionService.java:528) org.apache.usergrid.services.AbstractService.invokeItemsWithQuery(AbstractService.java:703) org.apache.usergrid.services.AbstractService.invoke(AbstractService.java:625) org.apache.usergrid.services.AbstractService.invoke(AbstractService.java:544) org.apache.usergrid.services.ServiceRequest.execute(ServiceRequest.java:226) org.apache.usergrid.services.ServiceRequest.execute(ServiceRequest.java:193) org.apache.usergrid.rest.applications.ServiceResource.executeServiceRequest(ServiceResource.java:251) org.apache.usergrid.rest.applications.ServiceResource.executeDelete(ServiceResource.java:437) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:601) com.sun.jersey.spi.container.JavaMethodInvokerFactory$1.invoke(JavaMethodInvokerFactory.java:60) com.sun.jersey.server.impl.model.method.dispatch.AbstractResourceMethodDispatchProvider$TypeOutInvoker._dispatch(AbstractResourceMethodDispatchProvider.java:185) com.sun.jersey.server.impl.model.method.dispatch.ResourceJavaMethodDispatcher.dispatch(ResourceJavaMethodDispatcher.java:75) com.sun.jersey.server.impl.uri.rules.HttpMethodRule.accept(HttpMethodRule.java:302) com.sun.jersey.server.impl.uri.rules.SubLocatorRule.accept(SubLocatorRule.java:137) com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147) com.sun.jersey.server.impl.uri.rules.SubLocatorRule.accept(SubLocatorRule.java:137) com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147) com.sun.jersey.server.impl.uri.rules.SubLocatorRule.accept(SubLocatorRule.java:137) com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147) com.sun.jersey.server.impl.uri.rules.ResourceClassRule.accept(ResourceClassRule.java:108) com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147) com.sun.jersey.server.impl.uri.rules.RootResourceClassesRule.accept(RootResourceClassesRule.java:84) com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1542) com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1473) com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1419) com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1409) com.sun.jersey.spi.container.servlet.WebComponent.service(WebComponent.java:409) com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:540) com.sun.jersey.spi.container.servlet.ServletContainer.doFilter(ServletContainer.java:909) com.sun.jersey.spi.container.servlet.ServletContainer.doFilter(ServletContainer.java:857) com.sun.jersey.spi.container.servlet.ServletContainer.doFilter(ServletContainer.java:811) org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241) org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208) org.apache.shiro.web.servlet.AbstractShiroFilter.executeChain(AbstractShiroFilter.java:449) org.apache.shiro.web.servlet.AbstractShiroFilter$1.call(AbstractShiroFilter.java:365) org.apache.shiro.subject.support.SubjectCallable.doCall(SubjectCallable.java:90) org.apache.shiro.subject.support.SubjectCallable.call(SubjectCallable.java:83) org.apache.shiro.subject.support.DelegatingSubject.execute(DelegatingSubject.java:383) org.apache.shiro.web.servlet.AbstractShiroFilter.doFilterInternal(AbstractShiroFilter.java:362) org.apache.shiro.web.servlet.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:125) org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:346) org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:259) org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241) org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208) org.apache.usergrid.rest.filters.ContentTypeFilter.doFilter(ContentTypeFilter.java:92) org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241) org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208) org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:220) org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:122) org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:503) org.apache.catalina.core.StandardHostValve....
Story points: 3