Project: springxd
Issue XD-2720: Frequent connection pool errors with multi-partitioned jdbchdfs jobs
I'm running a jdbchdfs job with 8 partitions and 2 containers. Some steps complete ok while some (3-4 on average) fail with a connection pool error (see below). This happens with a decent size table (1.8M rows). I tried two different databases - Oracle 11g on a separate server and MySQL running locally where the XD containers where running. Same pattern with both databases. {code} 2015-02-13 12:21:33,436 1.1.0.RELEASE ERROR inbound.files3.0-redis:queue-inbound-channel-adapter1 step.AbstractStep - Encountered an error executing step step1 in job files3 org.springframework.beans.factory.BeanCreationException: Error creating bean with name 'scopedTarget.itemReader' defined in file [/home/trisberg/Test/spring-xd-1.1.0.RELEASE/xd/modules/job/jdbchdfs/config/jdbchdfs.xml]: Invocation of init method failed; nested exception is org.springframework.jdbc.support.MetaDataAccessException: Could not get Connection for extracting meta data; nested exception is org.springframework.jdbc.CannotGetJdbcConnectionException: Could not get JDBC Connection; nested exception is java.sql.SQLException: Failed to validate a newly established connection. org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.initializeBean(AbstractAutowireCapableBeanFactory.java:1566) org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.doCreateBean(AbstractAutowireCapableBeanFactory.java:539) org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBean(AbstractAutowireCapableBeanFactory.java:476) org.springframework.beans.factory.support.AbstractBeanFactory$2.getObject(AbstractBeanFactory.java:342) org.springframework.batch.core.scope.StepScope.get(StepScope.java:110) org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBeanFactory.java:337) org.springframework.beans.factory.support.AbstractBeanFactory.getBean(AbstractBeanFactory.java:194) org.springframework.aop.target.SimpleBeanTargetSource.getTarget(SimpleBeanTargetSource.java:35) org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:187) com.sun.proxy.$Proxy90.open(Unknown Source) org.springframework.batch.item.support.CompositeItemStream.open(CompositeItemStream.java:96) org.springframework.batch.core.step.tasklet.TaskletStep.open(TaskletStep.java:310) org.springframework.batch.core.step.AbstractStep.execute(AbstractStep.java:195) org.springframework.batch.integration.partition.StepExecutionRequestHandler.handle(StepExecutionRequestHandler.java:64) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:606) org.springframework.expression.spel.support.ReflectiveMethodExecutor.execute(ReflectiveMethodExecutor.java:112) org.springframework.expression.spel.ast.MethodReference.getValueInternal(MethodReference.java:102) org.springframework.expression.spel.ast.MethodReference.access$000(MethodReference.java:49) org.springframework.expression.spel.ast.MethodReference$MethodValueRef.getValue(MethodReference.java:342) org.springframework.expression.spel.ast.CompoundExpression.getValueInternal(CompoundExpression.java:88) org.springframework.expression.spel.ast.SpelNodeImpl.getTypedValue(SpelNodeImpl.java:131) org.springframework.expression.spel.standard.SpelExpression.getValue(SpelExpression.java:330) org.springframework.integration.util.AbstractExpressionEvaluator.evaluateExpression(AbstractExpressionEvaluator.java:164) org.springframework.integration.util.MessagingMethodInvokerHelper.processInternal(MessagingMethodInvokerHelper.java:276) org.springframework.integration.util.MessagingMethodInvokerHelper.process(MessagingMethodInvokerHelper.java:142) org.springframework.integration.handler.MethodInvokingMessageProcessor.processMessage(MethodInvokingMessageProcessor.java:75) org.springframework.integration.handler.ServiceActivatingHandler.handleRequestMessage(ServiceActivatingHandler.java:71) org.springframework.integration.handler.AbstractReplyProducingMessageHandler.handleMessageInternal(AbstractReplyProducingMessageHandler.java:99) org.springframework.integration.handler.AbstractMessageHandler.handleMessage(AbstractMessageHandler.java:78) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:606) org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:317) org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:190) org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:157) org.springframework.integration.monitor.SimpleMessageHandlerMetrics.handleMessage(SimpleMessageHandlerMetrics.java:107) org.springframework.integration.monitor.SimpleMessageHandlerMetrics.invoke(SimpleMessageHandlerMetrics.java:87) org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179) org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:207) com.sun.proxy.$Proxy94.handleMessage(Unknown Source) org.springframework.integration.dispatcher.AbstractDispatcher.tryOptimizedDispatch(AbstractDispatcher.java:116) org.springframework.integration.dispatcher.UnicastingDispatcher.doDispatch(UnicastingDispatcher.java:101) org.springframework.integration.dispatcher.UnicastingDispatcher.dispatch(UnicastingDispatcher.java:97) org.springframework.integration.channel.AbstractSubscribableChannel.doSend(AbstractSubscribableChannel.java:77) org.springframework.integration.channel.AbstractMessageChannel.send(AbstractMessageChannel.java:277) org.springframework.integration.channel.AbstractMessageChannel.send(AbstractMessageChannel.java:239) sun.reflect.GeneratedMethodAccessor69.invoke(Unknown Source) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:606) org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:317) org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:190) org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:157) org.springframework.integration.monitor.DirectChannelMetrics.monitorSend(DirectChannelMetrics.java:114) org.springframework.integration.monitor.DirectChannelMetrics.doInvoke(DirectChannelMetrics.java:98) org.springframework.integration.monitor.DirectChannelMetrics.invoke(DirectChannelMetrics.java:92) org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179) org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:207) com.sun.proxy.$Proxy91.send(Unknown Source) org.springframework.messaging.core.GenericMessagingTemplate.doSend(GenericMessagingTemplate.java:115) org.springframework.messaging.core.GenericMessagingTemplate.doSend(GenericMessagingTemplate.java:45) org.springframework.messaging.core.AbstractMessageSendingTemplate.send(AbstractMessageSendingTemplate.java:95) org.springframework.integration.handler.AbstractMessageProducingHandler.sendOutput(AbstractMessageProducingHandler.java:248) org.springframework.integration.handler.AbstractMessageProducingHandler.produceOutput(AbstractMessageProducingHandler.java:171) org.springframework.integration.handler.AbstractMessageProducingHandler.sendOutputs(AbstractMessageProducingHandler.java:119) org.springframework.integration.handler.AbstractReplyProducingMessageHandler.handleMessageInternal(AbstractReplyProducingMessageHandler.java:105) org.springframework.integration.handler.AbstractMessageHandler.handleMessage(AbstractMessageHandler.java:78) org.springframework.integration.dispatcher.AbstractDispatcher.tryOptimizedDispatch(AbstractDispatcher.java:116) org.springframework.integration.dispatcher.UnicastingDispatcher.doDispatch(UnicastingDispatcher.java:101) org.springframework.integration.dispatcher.UnicastingDispatcher.dispatch(UnicastingDispatcher.java:97) org.springframework.integration.channel.AbstractSubscribableChannel.doSend(AbstractSubscribableChannel.java:77) org.springframework.integration.channel.AbstractMessageChannel.send(AbstractMessageChannel.java:277) org.springframework.xd.dirt.integration.redis.RedisMessageBus$1$1.doWithRetry(RedisMessageBus.java:267) org.springframework.xd.dirt.integration.redis.RedisMessageBus$1$1.doWithRetry(RedisMessageBus.java:263) org.springframework.retry.support.RetryTemplate.doExecute(RetryTemplate.java:263) org.springframework.retry.support.RetryTemplate.execute(RetryTemplate.java:168) org.springframework.xd.dirt.integration.redis.RedisMessageBus$1.doSend(RedisMessageBus.java:263) org.springframework.integration.channel.AbstractMessageChannel.send(AbstractMessageChannel.java:277) org.springframework.integration.channel.AbstractMessageChannel.send(AbstractMessageChannel.java:239) org.springframework.messaging.core.GenericMessagingTemplate.doSend(GenericMessagingTemplate.java:115) org.springframework.messaging.core.GenericMessagingTemplate.doSend(GenericMessagingTemplate.java:45) org.springframework.messaging.core.AbstractMessageSendingTemplate.send(AbstractMessageSendingTemplate.java:95) org.springframework.integration.endpoint.MessageProducerSupport.sendMessage(MessageProducerSupport.java:101) org.springframework.integration.redis.inbound.RedisQueueMessageDrivenEndpoint.popMessageAndSend(RedisQueueMessageDrivenEndpoint.java:220) org.springframework.integration.redis.inbound.RedisQueueMessageDrivenEndpoint.access$300(RedisQueueMessageDrivenEndpoint.java:50) org.springframework.integration.redis.inbound.RedisQueueMessageDrivenEndpoint$ListenerTask.run(RedisQueueMessageDrivenEndpoint.java:314) org.springframework.integration.util.ErrorHandlingTaskExecutor$1.run(ErrorHandlingTaskExecutor.java:52) java.lang.Thread.run(Thread.java:745) Caused by: org.springframework.jdbc.support.MetaDataAccessException: Could not get Connection for extracting meta data; nested exception is org.springframework.jdbc.CannotGetJdbcConnectionException: Could not get JDBC Connection; nested exception is java.sql.SQLException: Failed to validate a newly established connection. org.springframework.jdbc.support.JdbcUtils.extractDatabaseMetaData(JdbcUtils.java:302) org.springframework.jdbc.support.JdbcUtils.extractDatabaseMetaData(JdbcUtils.java:329) org.springframework.batch.support.DatabaseType.fromMetaData(DatabaseType.java:95) org.springframework.xd.jdbc.NamedColumnJdbcItemReaderFactory.afterPropertiesSet(NamedColumnJdbcItemReaderFactory.java:101) org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.invokeInitMethods(AbstractAutowireCapableBeanFactory.java:1625) org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.initializeBean(AbstractAutowireCapableBeanFactory.java:1562)	... 90 more Caused by: org.springframework.jdbc.CannotGetJdbcConnectionException: Could not get JDBC Connection; nested exception is java.sql.SQLException: Failed to validate a newly established connection. org.springframework.jdbc.datasource.DataSourceUtils.getConnection(DataSourceUtils.java:80) org.springframework.jdbc.support.JdbcUtils.extractDatabaseMetaData(JdbcUtils.java:289)	... 95 more Caused by: java.sql.SQLException: Failed to validate a newly established connection. org.apache.tomcat.jdbc.pool.ConnectionPool.borrowConnection(ConnectionPool.java:802) org.apache.tomcat.jdbc.pool.ConnectionPool.borrowConnection(ConnectionPool.java:617) org.apache.tomcat.jdbc.pool.ConnectionPool.getConnection(ConnectionPool.java:186) org.apache.tomcat.jdbc.pool.DataSourceProxy.getConnection(DataSourceProxy.java:127) org.springframework.jdbc.datasource.DataSourceUtils.doGetConnection(DataSourceUtils.java:111) org.springframework.jdbc.datasource.DataSourceUtils.getConnection(DataSourceUtils.java:77)	... 96 more {code}
Story points: 5