Project: springxd
Issue XD-1985: Packaging of Guava 17 results in failure to deploy mapreduce job to Hadoop 2.4 based distros
Trying to deploy the hashtagcount batch sample [1] to Hadoop 2.4.1 or Hortonworks HDP 2.1 fails with an IllegalAccessError exception. Looks like a Guava versioning issue - Swapping out guava-17.0.jar for guava-11.0.2.jar in the xd/lib directory solves it. Mark P suggested we try 16.0.1 which is what Curator uses and that seems to work as well. Looking into changing the build to not force 17.0 which is the IO platform version. http://upstream-tracker.org/java/compat_reports/guava/16.0.1_to_17.0/src_compat_report.html I get the following exception: {code} 16:42:22,214 INFO Deployer server.JobDeploymentListener - Deployment status for job 'hashtagCountJob': DeploymentStatus{state=deployed} 16:42:27,315 WARN task-scheduler-2 mapreduce.JobSubmitter - No job jar file set. User classes may not be found. See Job or Job#setJar(String). 16:42:27,325 ERROR task-scheduler-2 step.AbstractStep - Encountered an error executing step hashtagcount in job hashtagCountJob java.lang.IllegalAccessError: tried to access method com.google.common.base.Stopwatch.()V from class org.apache.hadoop.mapreduce.lib.input.FileInputFormat org.apache.hadoop.mapreduce.lib.input.FileInputFormat.getSplits(FileInputFormat.java:369) org.apache.hadoop.mapreduce.JobSubmitter.writeNewSplits(JobSubmitter.java:493) org.apache.hadoop.mapreduce.JobSubmitter.writeSplits(JobSubmitter.java:510) org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:394) org.apache.hadoop.mapreduce.Job$10.run(Job.java:1285) org.apache.hadoop.mapreduce.Job$10.run(Job.java:1282) java.security.AccessController.doPrivileged(Native Method) javax.security.auth.Subject.doAs(Subject.java:415) org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1556) org.apache.hadoop.mapreduce.Job.submit(Job.java:1282) org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:1303) org.apache.hadoop.mapreduce.Job$$FastClassBySpringCGLIB$$a048cbfe.invoke() org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:204) org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:708) org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:157) org.springframework.aop.support.DelegatingIntroductionInterceptor.doProceed(DelegatingIntroductionInterceptor.java:133) org.springframework.aop.support.DelegatingIntroductionInterceptor.invoke(DelegatingIntroductionInterceptor.java:121) org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179) org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:644) org.apache.hadoop.mapreduce.Job$$EnhancerBySpringCGLIB$$875ec891.waitForCompletion() org.springframework.data.hadoop.mapreduce.JobExecutor$2.run(JobExecutor.java:199) org.springframework.core.task.SyncTaskExecutor.execute(SyncTaskExecutor.java:50) org.springframework.data.hadoop.mapreduce.JobExecutor.startJobs(JobExecutor.java:170) org.springframework.data.hadoop.batch.mapreduce.JobTasklet.execute(JobTasklet.java:90) org.springframework.batch.core.step.tasklet.TaskletStep$ChunkTransactionCallback.doInTransaction(TaskletStep.java:406) org.springframework.batch.core.step.tasklet.TaskletStep$ChunkTransactionCallback.doInTransaction(TaskletStep.java:330) org.springframework.transaction.support.TransactionTemplate.execute(TransactionTemplate.java:133) org.springframework.batch.core.step.tasklet.TaskletStep$2.doInChunkContext(TaskletStep.java:271) org.springframework.batch.core.scope.context.StepContextRepeatCallback.doInIteration(StepContextRepeatCallback.java:77) org.springframework.batch.repeat.support.RepeatTemplate.getNextResult(RepeatTemplate.java:368) org.springframework.batch.repeat.support.RepeatTemplate.executeInternal(RepeatTemplate.java:215) org.springframework.batch.repeat.support.RepeatTemplate.iterate(RepeatTemplate.java:144) org.springframework.batch.core.step.tasklet.TaskletStep.doExecute(TaskletStep.java:257) org.springframework.batch.core.step.AbstractStep.execute(AbstractStep.java:198) org.springframework.batch.core.job.SimpleStepHandler.handleStep(SimpleStepHandler.java:148) org.springframework.batch.core.job.flow.JobFlowExecutor.executeStep(JobFlowExecutor.java:64) org.springframework.batch.core.job.flow.support.state.StepState.handle(StepState.java:67) org.springframework.batch.core.job.flow.support.SimpleFlow.resume(SimpleFlow.java:162) org.springframework.batch.core.job.flow.support.SimpleFlow.start(SimpleFlow.java:141) org.springframework.batch.core.job.flow.FlowJob.doExecute(FlowJob.java:134) org.springframework.batch.core.job.AbstractJob.execute(AbstractJob.java:304) org.springframework.batch.core.launch.support.SimpleJobLauncher$1.run(SimpleJobLauncher.java:135) org.springframework.core.task.SyncTaskExecutor.execute(SyncTaskExecutor.java:50) org.springframework.batch.core.launch.support.SimpleJobLauncher.run(SimpleJobLauncher.java:128) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:606) org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:317) org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:190) org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:157) org.springframework.batch.core.configuration.annotation.SimpleBatchConfiguration$PassthruAdvice.invoke(SimpleBatchConfiguration.java:127) org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179) org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:207) com.sun.proxy.$Proxy44.run(Unknown Source) org.springframework.batch.integration.launch.JobLaunchingMessageHandler.launch(JobLaunchingMessageHandler.java:50) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:606) org.springframework.expression.spel.support.ReflectiveMethodExecutor.execute(ReflectiveMethodExecutor.java:63) org.springframework.expression.spel.ast.MethodReference.getValueInternal(MethodReference.java:122) org.springframework.expression.spel.ast.MethodReference.access$000(MethodReference.java:44) org.springframework.expression.spel.ast.MethodReference$MethodValueRef.getValue(MethodReference.java:258) org.springframework.expression.spel.ast.CompoundExpression.getValueInternal(CompoundExpression.java:84) org.springframework.expression.spel.ast.SpelNodeImpl.getTypedValue(SpelNodeImpl.java:114) org.springframework.expression.spel.standard.SpelExpression.getValue(SpelExpression.java:111) org.springframework.integration.util.AbstractExpressionEvaluator.evaluateExpression(AbstractExpressionEvaluator.java:159) org.springframework.integration.util.MessagingMethodInvokerHelper.processInternal(MessagingMethodInvokerHelper.java:268) org.springframework.integration.util.MessagingMethodInvokerHelper.process(MessagingMethodInvokerHelper.java:142) org.springframework.integration.handler.MethodInvokingMessageProcessor.processMessage(MethodInvokingMessageProcessor.java:75) org.springframework.integration.handler.ServiceActivatingHandler.handleRequestMessage(ServiceActivatingHandler.java:71) org.springframework.integration.handler.AbstractReplyProducingMessageHandler.handleMessageInternal(AbstractReplyProducingMessageHandler.java:170) org.springframework.integration.handler.AbstractMessageHandler.handleMessage(AbstractMessageHandler.java:78) sun.reflect.GeneratedMethodAccessor98.invoke(Unknown Source) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:606) org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:317) org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:190) org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:157) org.springframework.integration.monitor.SimpleMessageHandlerMetrics.handleMessage(SimpleMessageHandlerMetrics.java:106) org.springframework.integration.monitor.SimpleMessageHandlerMetrics.invoke(SimpleMessageHandlerMetrics.java:86) org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179) org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:207) com.sun.proxy.$Proxy125.handleMessage(Unknown Source) org.springframework.integration.dispatcher.AbstractDispatcher.tryOptimizedDispatch(AbstractDispatcher.java:116) org.springframework.integration.dispatcher.UnicastingDispatcher.doDispatch(UnicastingDispatcher.java:101) org.springframework.integration.dispatcher.UnicastingDispatcher.dispatch(UnicastingDispatcher.java:97) org.springframework.integration.channel.AbstractSubscribableChannel.doSend(AbstractSubscribableChannel.java:77) org.springframework.integration.channel.AbstractMessageChannel.send(AbstractMessageChannel.java:255) org.springframework.integration.channel.AbstractMessageChannel.send(AbstractMessageChannel.java:223) sun.reflect.GeneratedMethodAccessor97.invoke(Unknown Source) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:606) org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:317) org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:190) org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:157) org.springframework.integration.monitor.DirectChannelMetrics.monitorSend(DirectChannelMetrics.java:113) org.springframework.integration.monitor.DirectChannelMetrics.doInvoke(DirectChannelMetrics.java:97) org.springframework.integration.monitor.DirectChannelMetrics.invoke(DirectChannelMetrics.java:91) org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179) org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:207) com.sun.proxy.$Proxy123.send(Unknown Source) org.springframework.messaging.core.GenericMessagingTemplate.doSend(GenericMessagingTemplate.java:109) org.springframework.messaging.core.GenericMessagingTemplate.doSend(GenericMessagingTemplate.java:44) org.springframework.messaging.core.AbstractMessageSendingTemplate.send(AbstractMessageSendingTemplate.java:94) org.springframework.integration.handler.AbstractReplyProducingMessageHandler.sendMessage(AbstractReplyProducingMessageHandler.java:260) org.springframework.integration.handler.AbstractReplyProducingMessageHandler.sendReplyMessage(AbstractReplyProducingMessageHandler.java:241) org.springframework.integration.handler.AbstractReplyProducingMessageHandler.produceReply(AbstractReplyProducingMessageHandler.java:205) org.springframework.integration.handler.AbstractReplyProducingMessageHandler.handleResult(AbstractReplyProducingMessageHandler.java:199) org.springframework.integration.handler.AbstractReplyProducingMessageHandler.handleMessageInternal(AbstractReplyProducingMessageHandler.java:177) org.springframework.integration.handler.AbstractMessageHandler.handleMessage(AbstractMessageHandler.java:78) sun.reflect.GeneratedMethodAccessor98.invoke(Unknown Source) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:606) org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:317) org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:190) org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:157) org.springframework.integration.monitor.SimpleMessageHandlerMetrics.handleMessage(SimpleMessageHandlerMetrics.java:106) org.springframework.integration.monitor.SimpleMessageHandlerMetrics.invoke(SimpleMessageHandlerMetrics.java:86) org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179) org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:207) com.sun.proxy.$Proxy125.handleMessage(Unknown Source) org.springframework.integration.dispatcher.AbstractDispatcher.tryOptimizedDispatch(AbstractDispatcher.java:116) org.springframework.integration.dispatcher.UnicastingDispatcher.doDispatch(UnicastingDispatcher.java:101) org.springframework.integration.dispatcher.UnicastingDispatcher.dispatch(UnicastingDispatcher.java:97) org.springframework.integration.channel.AbstractSubscribableChannel.doSend(AbstractSubscribableChannel.java:77) org.springframework.integration.channel.AbstractMessageChannel.send(AbstractMessageChannel.java:255) org.springframework.integration.channel.AbstractMessageChannel.send(AbstractMessageChannel.java:223) sun.reflect.GeneratedMethodAccessor97.invoke(Unknown Source) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:606) org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:317) org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:190) org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:157) org.springframework.integration.monitor.DirectChannelMetrics.monitorSend(DirectChannelMetrics.java:113) org.springframework.integration.monitor.DirectChannelMetrics.doInvoke(DirectChannelMetrics.java:97) org.springframework.integration.monitor.DirectChannelMetrics.invoke(DirectChannelMetrics.java:91) org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179) org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:207) com.sun.proxy.$Proxy123.send(Unknown Source) org.springframework.messaging.core.GenericMessagingTemplate.doSend(GenericMessagingTemplate.java:109) org.springframework.messaging.core.GenericMessagingTemplate.doSend(GenericMessagingTemplate.java:44) org.springframework.messaging.core.AbstractMessageSendingTemplate.send(AbstractMessageSendingTemplate.java:94) org.springframework.integration.handler.AbstractReplyProducingMessageHandler.sendMessage(AbstractReplyProducingMessageHandler.java:260) org.springframework.integration.handler.AbstractReplyProducingMessageHandler.sendReplyMessage(AbstractReplyProducingMessageHandler.java:241) org.springframework.integration.handler.AbstractReplyProducingMessageHandler.produceReply(AbstractReplyProducingMessageHandler.java:205) org.springframework.integration.handler.AbstractReplyProducingMessageHandler.handleResult(AbstractReplyProducingMessageHandler.java:199) org.springframework.integration.handler.AbstractReplyProducingMessageHandler.handleMessageInternal(AbstractReplyProducingMessageHandler.java:177) org.springframework.integration.handler.AbstractMessageHandler.handleMessage(AbstractMessageHandler.java:78) org.springframework.integration.endpoint.PollingConsumer.handleMessage(PollingConsumer.java:74) org.springframework.integration.endpoint.AbstractPollingEndpoint.doPoll(AbstractPollingEndpoint.java:205) org.springframework.integration.endpoint.AbstractPollingEndpoint.access$000(AbstractPollingEndpoint.java:55) org.springframework.integration.endpoint.AbstractPollingEndpoint$1.call(AbstractPollingEndpoint.java:149) org.springframework.integration.endpoint.AbstractPollingEndpoint$1.call(AbstractPollingEndpoint.java:146) org.springframework.integration.endpoint.AbstractPollingEndpoint$Poller$1.run(AbstractPollingEndpoint.java:284) org.springframework.integration.util.ErrorHandlingTaskExecutor$1.run(ErrorHandlingTaskExecutor.java:52) org.springframework.core.task.SyncTaskExecutor.execute(SyncTaskExecutor.java:50) org.springframework.integration.util.ErrorHandlingTaskExecutor.execute(ErrorHandlingTaskExecutor.java:49) org.springframework.integration.endpoint.AbstractPollingEndpoint$Poller.run(AbstractPollingEndpoint.java:278) org.springframework.scheduling.support.DelegatingErrorHandlingRunnable.run(DelegatingErrorHandlingRunnable.java:54) org.springframework.scheduling.concurrent.ReschedulingRunnable.run(ReschedulingRunnable.java:81) java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) java.util.concurrent.FutureTask.run(FutureTask.java:262) java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:178) java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:292) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) {code} [1] https://github.com/spring-projects/spring-xd-samples
Story points: 5