Project: springxd
Issue XD-3642: Spring XD unable to save metadata for long steps
Hey Guys, We are facing an annoying problem and I can't figure out how solve it. For some reason when we run a step that takes long time (4 hours or more), spring XD is unable to save the metadata and throws the following error. 2015-10-25T09:49:10-0400 1.2.0.RELEASE ERROR task-scheduler-2 step.AbstractStep - Encountered an error executing step ingest-logs in job ingest-logs-flow org.springframework.orm.jpa.JpaSystemException: commit failed; nested exception is org.hibernate.TransactionException: commit failed org.springframework.orm.jpa.vendor.HibernateJpaDialect.convertHibernateAccessException(HibernateJpaDialect.java:244) ~[na:na] org.springframework.orm.jpa.vendor.HibernateJpaDialect.translateExceptionIfPossible(HibernateJpaDialect.java:155) ~[na:na] org.springframework.orm.jpa.JpaTransactionManager.doCommit(JpaTransactionManager.java:521) ~[na:na] org.springframework.transaction.support.AbstractPlatformTransactionManager.processCommit(AbstractPlatformTransactionManager.java:757) ~[spring-tx-4.1.6.RELEASE.jar:4.1.6.RELEASE] org.springframework.transaction.support.AbstractPlatformTransactionManager.commit(AbstractPlatformTransactionManager.java:726) ~[spring-tx-4.1.6.RELEASE.jar:4.1.6.RELEASE] org.springframework.transaction.support.TransactionTemplate.execute(TransactionTemplate.java:150) ~[spring-tx-4.1.6.RELEASE.jar:4.1.6.RELEASE] org.springframework.batch.core.step.tasklet.TaskletStep$2.doInChunkContext(TaskletStep.java:271) ~[spring-batch-core-3.0.3.RELEASE.jar:3.0.3.RELEASE] org.springframework.batch.core.scope.context.StepContextRepeatCallback.doInIteration(StepContextRepeatCallback.java:77) ~[spring-batch-core-3.0.3.RELEASE.jar:3.0.3.RELEASE] org.springframework.batch.repeat.support.RepeatTemplate.getNextResult(RepeatTemplate.java:368) ~[spring-batch-infrastructure-3.0.3.RELEASE.jar:3.0.3.RELEASE] org.springframework.batch.repeat.support.RepeatTemplate.executeInternal(RepeatTemplate.java:215) ~[spring-batch-infrastructure-3.0.3.RELEASE.jar:3.0.3.RELEASE] org.springframework.batch.repeat.support.RepeatTemplate.iterate(RepeatTemplate.java:144) ~[spring-batch-infrastructure-3.0.3.RELEASE.jar:3.0.3.RELEASE] org.springframework.batch.core.step.tasklet.TaskletStep.doExecute(TaskletStep.java:257) ~[spring-batch-core-3.0.3.RELEASE.jar:3.0.3.RELEASE] org.springframework.batch.core.step.AbstractStep.execute(AbstractStep.java:198) ~[spring-batch-core-3.0.3.RELEASE.jar:3.0.3.RELEASE] org.springframework.batch.core.job.SimpleStepHandler.handleStep(SimpleStepHandler.java:148) [spring-batch-core-3.0.3.RELEASE.jar:3.0.3.RELEASE] org.springframework.batch.core.job.flow.JobFlowExecutor.executeStep(JobFlowExecutor.java:64) [spring-batch-core-3.0.3.RELEASE.jar:3.0.3.RELEASE] org.springframework.batch.core.job.flow.support.state.StepState.handle(StepState.java:67) [spring-batch-core-3.0.3.RELEASE.jar:3.0.3.RELEASE] org.springframework.batch.core.job.flow.support.SimpleFlow.resume(SimpleFlow.java:165) [spring-batch-core-3.0.3.RELEASE.jar:3.0.3.RELEASE] org.springframework.batch.core.job.flow.support.SimpleFlow.start(SimpleFlow.java:144) [spring-batch-core-3.0.3.RELEASE.jar:3.0.3.RELEASE] org.springframework.batch.core.job.flow.FlowJob.doExecute(FlowJob.java:134) [spring-batch-core-3.0.3.RELEASE.jar:3.0.3.RELEASE] org.springframework.batch.core.job.AbstractJob.execute(AbstractJob.java:304) [spring-batch-core-3.0.3.RELEASE.jar:3.0.3.RELEASE] org.springframework.batch.core.launch.support.SimpleJobLauncher$1.run(SimpleJobLauncher.java:135) [spring-batch-core-3.0.3.RELEASE.jar:3.0.3.RELEASE] org.springframework.core.task.SyncTaskExecutor.execute(SyncTaskExecutor.java:50) [spring-core-4.1.6.RELEASE.jar:4.1.6.RELEASE] org.springframework.batch.core.launch.support.SimpleJobLauncher.run(SimpleJobLauncher.java:128) [spring-batch-core-3.0.3.RELEASE.jar:3.0.3.RELEASE] sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[na:1.7.0_67] sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) ~[na:1.7.0_67] sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.7.0_67] java.lang.reflect.Method.invoke(Method.java:606) ~[na:1.7.0_67] org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:317) [spring-aop-4.1.6.RELEASE.jar:4.1.6.RELEASE] org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:190) [spring-aop-4.1.6.RELEASE.jar:4.1.6.RELEASE] org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:157) [spring-aop-4.1.6.RELEASE.jar:4.1.6.RELEASE] org.springframework.batch.core.configuration.annotation.SimpleBatchConfiguration$PassthruAdvice.invoke(SimpleBatchConfiguration.java:127) [spring-batch-core-3.0.3.RELEASE.jar:3.0.3.RELEASE] org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179) [spring-aop-4.1.6.RELEASE.jar:4.1.6.RELEASE] org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:207) [spring-aop-4.1.6.RELEASE.jar:4.1.6.RELEASE] com.sun.proxy.$Proxy53.run(Unknown Source) [na:na] org.springframework.batch.integration.launch.JobLaunchingMessageHandler.launch(JobLaunchingMessageHandler.java:50) [spring-batch-integration-3.0.3.RELEASE.jar:3.0.3.RELEASE] sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[na:1.7.0_67] sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) ~[na:1.7.0_67] sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.7.0_67] java.lang.reflect.Method.invoke(Method.java:606) ~[na:1.7.0_67] org.springframework.expression.spel.support.ReflectiveMethodExecutor.execute(ReflectiveMethodExecutor.java:112) [spring-expression-4.1.6.RELEASE.jar:4.1.6.RELEASE] org.springframework.expression.spel.ast.MethodReference.getValueInternal(MethodReference.java:129) [spring-expression-4.1.6.RELEASE.jar:4.1.6.RELEASE] org.springframework.expression.spel.ast.MethodReference.access$000(MethodReference.java:49) [spring-expression-4.1.6.RELEASE.jar:4.1.6.RELEASE] org.springframework.expression.spel.ast.MethodReference$MethodValueRef.getValue(MethodReference.java:342) [spring-expression-4.1.6.RELEASE.jar:4.1.6.RELEASE] org.springframework.expression.spel.ast.CompoundExpression.getValueInternal(CompoundExpression.java:88) [spring-expression-4.1.6.RELEASE.jar:4.1.6.RELEASE] org.springframework.expression.spel.ast.SpelNodeImpl.getTypedValue(SpelNodeImpl.java:131) [spring-expression-4.1.6.RELEASE.jar:4.1.6.RELEASE] org.springframework.expression.spel.standard.SpelExpression.getValue(SpelExpression.java:330) [spring-expression-4.1.6.RELEASE.jar:4.1.6.RELEASE] org.springframework.integration.util.AbstractExpressionEvaluator.evaluateExpression(AbstractExpressionEvaluator.java:164) [spring-integration-core-4.1.5.RELEASE.jar:na] org.springframework.integration.util.MessagingMethodInvokerHelper.processInternal(MessagingMethodInvokerHelper.java:276) [spring-integration-core-4.1.5.RELEASE.jar:na] org.springframework.integration.util.MessagingMethodInvokerHelper.process(MessagingMethodInvokerHelper.java:142) [spring-integration-core-4.1.5.RELEASE.jar:na] org.springframework.integration.handler.MethodInvokingMessageProcessor.processMessage(MethodInvokingMessageProcessor.java:75) [spring-integration-core-4.1.5.RELEASE.jar:na] org.springframework.integration.handler.ServiceActivatingHandler.handleRequestMessage(ServiceActivatingHandler.java:71) [spring-integration-core-4.1.5.RELEASE.jar:na] org.springframework.integration.handler.AbstractReplyProducingMessageHandler.handleMessageInternal(AbstractReplyProducingMessageHandler.java:99) [spring-integration-core-4.1.5.RELEASE.jar:na] org.springframework.integration.handler.AbstractMessageHandler.handleMessage(AbstractMessageHandler.java:78) [spring-integration-core-4.1.5.RELEASE.jar:na] org.springframework.integration.dispatcher.AbstractDispatcher.tryOptimizedDispatch(AbstractDispatcher.java:116) [spring-integration-core-4.1.5.RELEASE.jar:na] org.springframework.integration.dispatcher.UnicastingDispatcher.doDispatch(UnicastingDispatcher.java:101) [spring-integration-core-4.1.5.RELEASE.jar:na] org.springframework.integration.dispatcher.UnicastingDispatcher.dispatch(UnicastingDispatcher.java:97) [spring-integration-core-4.1.5.RELEASE.jar:na] org.springframework.integration.channel.AbstractSubscribableChannel.doSend(AbstractSubscribableChannel.java:77) [spring-integration-core-4.1.5.RELEASE.jar:na] org.springframework.integration.channel.AbstractMessageChannel.send(AbstractMessageChannel.java:287) [spring-integration-core-4.1.5.RELEASE.jar:na] org.springframework.integration.channel.AbstractMessageChannel.send(AbstractMessageChannel.java:245) [spring-integration-core-4.1.5.RELEASE.jar:na] org.springframework.messaging.core.GenericMessagingTemplate.doSend(GenericMessagingTemplate.java:115) [spring-messaging-4.1.6.RELEASE.jar:4.1.6.RELEASE] org.springframework.messaging.core.GenericMessagingTemplate.doSend(GenericMessagingTemplate.java:45) [spring-messaging-4.1.6.RELEASE.jar:4.1.6.RELEASE] org.springframework.messaging.core.AbstractMessageSendingTemplate.send(AbstractMessageSendingTemplate.java:95) [spring-messaging-4.1.6.RELEASE.jar:4.1.6.RELEASE] org.springframework.integration.handler.AbstractMessageProducingHandler.sendOutput(AbstractMessageProducingHandler.java:231) [spring-integration-core-4.1.5.RELEASE.jar:na] org.springframework.integration.handler.AbstractMessageProducingHandler.produceOutput(AbstractMessageProducingHandler.java:154) [spring-integration-core-4.1.5.RELEASE.jar:na] org.springframework.integration.handler.AbstractMessageProducingHandler.sendOutputs(AbstractMessageProducingHandler.java:102) [spring-integration-core-4.1.5.RELEASE.jar:na] org.springframework.integration.handler.AbstractReplyProducingMessageHandler.handleMessageInternal(AbstractReplyProducingMessageHandler.java:105) [spring-integration-core-4.1.5.RELEASE.jar:na] org.springframework.integration.handler.AbstractMessageHandler.handleMessage(AbstractMessageHandler.java:78) [spring-integration-core-4.1.5.RELEASE.jar:na] org.springframework.integration.dispatcher.AbstractDispatcher.tryOptimizedDispatch(AbstractDispatcher.java:116) [spring-integration-core-4.1.5.RELEASE.jar:na] org.springframework.integration.dispatcher.UnicastingDispatcher.doDispatch(UnicastingDispatcher.java:101) [spring-integration-core-4.1.5.RELEASE.jar:na] org.springframework.integration.dispatcher.UnicastingDispatcher.dispatch(UnicastingDispatcher.java:97) [spring-integration-core-4.1.5.RELEASE.jar:na] org.springframework.integration.channel.AbstractSubscribableChannel.doSend(AbstractSubscribableChannel.java:77) [spring-integration-core-4.1.5.RELEASE.jar:na] org.springframework.integration.channel.AbstractMessageChannel.send(AbstractMessageChannel.java:287) [spring-integration-core-4.1.5.RELEASE.jar:na] org.springframework.integration.channel.AbstractMessageChannel.send(AbstractMessageChannel.java:245) [spring-integration-core-4.1.5.RELEASE.jar:na] org.springframework.messaging.core.GenericMessagingTemplate.doSend(GenericMessagingTemplate.java:115) [spring-messaging-4.1.6.RELEASE.jar:4.1.6.RELEASE] org.springframework.messaging.core.GenericMessagingTemplate.doSend(GenericMessagingTemplate.java:45) [spring-messaging-4.1.6.RELEASE.jar:4.1.6.RELEASE] org.springframework.messaging.core.AbstractMessageSendingTemplate.send(AbstractMessageSendingTemplate.java:95) [spring-messaging-4.1.6.RELEASE.jar:4.1.6.RELEASE] org.springframework.integration.handler.AbstractMessageProducingHandler.sendOutput(AbstractMessageProducingHandler.java:231) [spring-integration-core-4.1.5.RELEASE.jar:na] org.springframework.integration.handler.AbstractMessageProducingHandler.produceOutput(AbstractMessageProducingHandler.java:154) [spring-integration-core-4.1.5.RELEASE.jar:na] org.springframework.integration.handler.AbstractMessageProducingHandler.sendOutputs(AbstractMessageProducingHandler.java:102) [spring-integration-core-4.1.5.RELEASE.jar:na] org.springframework.integration.handler.AbstractReplyProducingMessageHandler.handleMessageInternal(AbstractReplyProducingMessageHandler.java:105) [spring-integration-core-4.1.5.RELEASE.jar:na] org.springframework.integration.handler.AbstractMessageHandler.handleMessage(AbstractMessageHandler.java:78) [spring-integration-core-4.1.5.RELEASE.jar:na] org.springframework.integration.endpoint.PollingConsumer.handleMessage(PollingConsumer.java:74) [spring-integration-core-4.1.5.RELEASE.jar:na] org.springframework.integration.endpoint.AbstractPollingEndpoint.doPoll(AbstractPollingEndpoint.java:219) [spring-integration-core-4.1.5.RELEASE.jar:na] org.springframework.integration.endpoint.AbstractPollingEndpoint.access$000(AbstractPollingEndpoint.java:55) [spring-integration-core-4.1.5.RELEASE.jar:na] org.springframework.integration.endpoint.AbstractPollingEndpoint$1.call(AbstractPollingEndpoint.java:149) [spring-integration-core-4.1.5.RELEASE.jar:na] org.springframework.integration.endpoint.AbstractPollingEndpoint$1.call(AbstractPollingEndpoint.java:146) [spring-integration-core-4.1.5.RELEASE.jar:na] org.springframework.integration.endpoint.AbstractPollingEndpoint$Poller$1.run(AbstractPollingEndpoint.java:298) [spring-integration-core-4.1.5.RELEASE.jar:na] org.springframework.integration.util.ErrorHandlingTaskExecutor$1.run(ErrorHandlingTaskExecutor.java:52) [spring-integration-core-4.1.5.RELEASE.jar:na] org.springframework.core.task.SyncTaskExecutor.execute(SyncTaskExecutor.java:50) [spring-core-4.1.6.RELEASE.jar:4.1.6.RELEASE] org.springframework.integration.util.ErrorHandlingTaskExecutor.execute(ErrorHandlingTaskExecutor.java:49) [spring-integration-core-4.1.5.RELEASE.jar:na] org.springframework.integration.endpoint.AbstractPollingEndpoint$Poller.run(AbstractPollingEndpoint.java:292) [spring-integration-core-4.1.5.RELEASE.jar:na] org.springframework.scheduling.support.DelegatingErrorHandlingRunnable.run(DelegatingErrorHandlingRunnable.java:54) [spring-context-4.1.6.RELEASE.jar:4.1.6.RELEASE] org.springframework.scheduling.concurrent.ReschedulingRunnable.run(ReschedulingRunnable.java:81) [spring-context-4.1.6.RELEASE.jar:4.1.6.RELEASE] java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) [na:1.7.0_67] java.util.concurrent.FutureTask.run(FutureTask.java:262) [na:1.7.0_67] java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:178) [na:1.7.0_67] java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:292) [na:1.7.0_67] java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) [na:1.7.0_67] java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) [na:1.7.0_67] java.lang.Thread.run(Thread.java:745) [na:1.7.0_67] Caused by: org.hibernate.TransactionException: commit failed org.hibernate.engine.transaction.spi.AbstractTransactionImpl.commit(AbstractTransactionImpl.java:187) ~[na:na] org.hibernate.jpa.internal.TransactionImpl.commit(TransactionImpl.java:77) ~[na:na] org.springframework.orm.jpa.JpaTransactionManager.doCommit(JpaTransactionManager.java:517) ~[na:na]	... 97 common frames omitted Caused by: org.hibernate.TransactionException: unable to commit against JDBC connection org.hibernate.engine.transaction.internal.jdbc.JdbcTransaction.doCommit(JdbcTransaction.java:116) ~[na:na] org.hibernate.engine.transaction.spi.AbstractTransactionImpl.commit(AbstractTransactionImpl.java:180) ~[na:na]	... 99 common frames omitted Caused by: com.mysql.jdbc.exceptions.jdbc4.MySQLNonTransientConnectionException: Communications link failure during commit(). Transaction resolution unknown. sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[na:1.7.0_67] sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57) ~[na:1.7.0_67] sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[na:1.7.0_67] java.lang.reflect.Constructor.newInstance(Constructor.java:526) ~[na:1.7.0_67] com.mysql.jdbc.Util.handleNewInstance(Util.java:389) ~[mysql-connector-java-5.1.34.jar:5.1.34] com.mysql.jdbc.Util.getInstance(Util.java:372) ~[mysql-connector-java-5.1.34.jar:5.1.34] com.mysql.jdbc.SQLError.createSQLException(SQLError.java:958) ~[mysql-connector-java-5.1.34.jar:5.1.34] com.mysql.jdbc.SQLError.createSQLException(SQLError.java:937) ~[mysql-connector-java-5.1.34.jar:5.1.34] com.mysql.jdbc.SQLError.createSQLException(SQLError.java:926) ~[mysql-connector-java-5.1.34.jar:5.1.34] com.mysql.jdbc.SQLError.createSQLException(SQLError.java:872) ~[mysql-connector-java-5.1.34.jar:5.1.34] com.mysql.jdbc.ConnectionImpl.commit(ConnectionImpl.java:1616) ~[mysql-connector-java-5.1.34.jar:5.1.34] org.apache.commons.dbcp.DelegatingConnection.commit(DelegatingConnection.java:301) ~[commons-dbcp-1.4.jar:1.4] org.apache.commons.dbcp.PoolingDataSource$PoolGuardConnectionWrapper.commit(PoolingDataSource.java:200) ~[commons-dbcp-1.4.jar:1.4] org.hibernate.engine.transaction.internal.jdbc.JdbcTransaction.doCommit(JdbcTransaction.java:112) ~[na:na]	... 100 common frames omitted 2015-10-25T09:49:10-0400 1.2.0.RELEASE ERROR task-scheduler-2 step.AbstractStep - Encountered an error saving batch meta data for step ingest-logs in job ingest-logs-flow. This job is now in an unknown state and should not be restarted. org.springframework.dao.OptimisticLockingFailureException: Attempt to update step execution id=8 with wrong version (1), where current version is 2 org.springframework.batch.core.repository.dao.JdbcStepExecutionDao.updateStepExecution(JdbcStepExecutionDao.java:255) ~[spring-batch-core-3.0.3.RELEASE.jar:3.0.3.RELEASE] We are using a MySQL database with the following configurations. spring: datasource: url: jdbc:mysql://hosttomysql:3306/singnode username: admin password: admin driverClassName: com.mysql.jdbc.Driver validationQuery: select 1 testOnBorrow: true If you guys know this issue or have some ideas how solve, please let me know. Thanks in advance. Héctor
Story points: 3