Project: springxd
Issue XD-2074: Modules that depend on HttpClient fail when running on YARN on Hadoop 2.4.x and later
Trying to run a twitterstream on YARN on HDP 2.1 and get the following: {code} 14/08/08 12:12:50 ERROR server.ContainerRegistrar: Exception deploying module org.springframework.beans.factory.BeanCreationException: Error creating bean with name 'org.springframework.integration.x.twitter.TwitterStreamChannelAdapter#0' defined in URL [file:./spring-xd-yarn-1.0.0.RELEASE.zip/modules/source/twitterstream/config/twitterstream.xml]: Cannot resolve reference to bean 'twitterTemplate' while setting constructor argument; nested exception is org.springframework.beans.factory.BeanCreationException: Error creating bean with name 'twitterTemplate' defined in URL [file:./spring-xd-yarn-1.0.0.RELEASE.zip/modules/source/twitterstream/config/twitterstream.xml]: Instantiation of bean failed; nested exception is org.springframework.beans.BeanInstantiationException: Could not instantiate bean class [org.springframework.social.twitter.api.impl.TwitterTemplate]: Constructor threw exception; nested exception is java.lang.NoClassDefFoundError: org/apache/http/impl/client/HttpClients org.springframework.beans.factory.support.BeanDefinitionValueResolver.resolveReference(BeanDefinitionValueResolver.java:336) org.springframework.beans.factory.support.BeanDefinitionValueResolver.resolveValueIfNecessary(BeanDefinitionValueResolver.java:108) org.springframework.beans.factory.support.ConstructorResolver.resolveConstructorArguments(ConstructorResolver.java:646) org.springframework.beans.factory.support.ConstructorResolver.autowireConstructor(ConstructorResolver.java:140) org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.autowireConstructor(AbstractAutowireCapableBeanFactory.java:1114) org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBeanInstance(AbstractAutowireCapableBeanFactory.java:1017) org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.doCreateBean(AbstractAutowireCapableBeanFactory.java:504) org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBean(AbstractAutowireCapableBeanFactory.java:475) org.springframework.beans.factory.support.AbstractBeanFactory$1.getObject(AbstractBeanFactory.java:302) org.springframework.beans.factory.support.DefaultSingletonBeanRegistry.getSingleton(DefaultSingletonBeanRegistry.java:228) org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBeanFactory.java:298) org.springframework.beans.factory.support.AbstractBeanFactory.getBean(AbstractBeanFactory.java:193) org.springframework.beans.factory.support.DefaultListableBeanFactory.preInstantiateSingletons(DefaultListableBeanFactory.java:703) org.springframework.context.support.AbstractApplicationContext.finishBeanFactoryInitialization(AbstractApplicationContext.java:760) org.springframework.context.support.AbstractApplicationContext.refresh(AbstractApplicationContext.java:482) org.springframework.boot.SpringApplication.refresh(SpringApplication.java:691) org.springframework.boot.SpringApplication.run(SpringApplication.java:320) org.springframework.boot.builder.SpringApplicationBuilder.run(SpringApplicationBuilder.java:142) org.springframework.xd.module.core.SimpleModule.initialize(SimpleModule.java:203) org.springframework.xd.dirt.module.ModuleDeployer.deploy(ModuleDeployer.java:98) org.springframework.xd.dirt.module.ModuleDeployer.deployAndStore(ModuleDeployer.java:88) org.springframework.xd.dirt.module.ModuleDeployer.deployAndStore(ModuleDeployer.java:78) org.springframework.xd.dirt.server.ContainerRegistrar.deployModule(ContainerRegistrar.java:231) org.springframework.xd.dirt.server.ContainerRegistrar.deployStreamModule(ContainerRegistrar.java:577) org.springframework.xd.dirt.server.ContainerRegistrar.onChildAdded(ContainerRegistrar.java:447) org.springframework.xd.dirt.server.ContainerRegistrar.access$800(ContainerRegistrar.java:95) org.springframework.xd.dirt.server.ContainerRegistrar$DeploymentListener.childEvent(ContainerRegistrar.java:826) org.apache.curator.framework.recipes.cache.PathChildrenCache$5.apply(PathChildrenCache.java:509) org.apache.curator.framework.recipes.cache.PathChildrenCache$5.apply(PathChildrenCache.java:503) org.apache.curator.framework.listen.ListenerContainer$1.run(ListenerContainer.java:92) com.google.common.util.concurrent.MoreExecutors$SameThreadExecutorService.execute(MoreExecutors.java:297) org.apache.curator.framework.listen.ListenerContainer.forEach(ListenerContainer.java:83) org.apache.curator.framework.recipes.cache.PathChildrenCache.callListeners(PathChildrenCache.java:500) org.apache.curator.framework.recipes.cache.EventOperation.invoke(EventOperation.java:35) org.apache.curator.framework.recipes.cache.PathChildrenCache$10.run(PathChildrenCache.java:762) java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) java.util.concurrent.FutureTask.run(FutureTask.java:262) java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) java.util.concurrent.FutureTask.run(FutureTask.java:262) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:744) Caused by: org.springframework.beans.factory.BeanCreationException: Error creating bean with name 'twitterTemplate' defined in URL [file:./spring-xd-yarn-1.0.0.RELEASE.zip/modules/source/twitterstream/config/twitterstream.xml]: Instantiation of bean failed; nested exception is org.springframework.beans.BeanInstantiationException: Could not instantiate bean class [org.springframework.social.twitter.api.impl.TwitterTemplate]: Constructor threw exception; nested exception is java.lang.NoClassDefFoundError: org/apache/http/impl/client/HttpClients org.springframework.beans.factory.support.ConstructorResolver.autowireConstructor(ConstructorResolver.java:278) org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.autowireConstructor(AbstractAutowireCapableBeanFactory.java:1114) org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBeanInstance(AbstractAutowireCapableBeanFactory.java:1017) org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.doCreateBean(AbstractAutowireCapableBeanFactory.java:504) org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBean(AbstractAutowireCapableBeanFactory.java:475) org.springframework.beans.factory.support.AbstractBeanFactory$1.getObject(AbstractBeanFactory.java:302) org.springframework.beans.factory.support.DefaultSingletonBeanRegistry.getSingleton(DefaultSingletonBeanRegistry.java:228) org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBeanFactory.java:298) org.springframework.beans.factory.support.AbstractBeanFactory.getBean(AbstractBeanFactory.java:193) org.springframework.beans.factory.support.BeanDefinitionValueResolver.resolveReference(BeanDefinitionValueResolver.java:328)	... 41 more Caused by: org.springframework.beans.BeanInstantiationException: Could not instantiate bean class [org.springframework.social.twitter.api.impl.TwitterTemplate]: Constructor threw exception; nested exception is java.lang.NoClassDefFoundError: org/apache/http/impl/client/HttpClients org.springframework.beans.BeanUtils.instantiateClass(BeanUtils.java:164) org.springframework.beans.factory.support.SimpleInstantiationStrategy.instantiate(SimpleInstantiationStrategy.java:125) org.springframework.beans.factory.support.ConstructorResolver.autowireConstructor(ConstructorResolver.java:270)	... 50 more Caused by: java.lang.NoClassDefFoundError: org/apache/http/impl/client/HttpClients org.springframework.http.client.HttpComponentsClientHttpRequestFactory.(HttpComponentsClientHttpRequestFactory.java:74) org.springframework.social.support.ClientHttpRequestFactorySelector$HttpComponentsClientRequestFactoryCreator$1.(ClientHttpRequestFactorySelector.java:77) org.springframework.social.support.ClientHttpRequestFactorySelector$HttpComponentsClientRequestFactoryCreator.createRequestFactory(ClientHttpRequestFactorySelector.java:77) org.springframework.social.support.ClientHttpRequestFactorySelector.getRequestFactory(ClientHttpRequestFactorySelector.java:52) org.springframework.social.oauth1.AbstractOAuth1ApiBinding.createRestTemplateWithCulledMessageConverters(AbstractOAuth1ApiBinding.java:188) org.springframework.social.oauth1.AbstractOAuth1ApiBinding.createRestTemplate(AbstractOAuth1ApiBinding.java:169) org.springframework.social.oauth1.AbstractOAuth1ApiBinding.(AbstractOAuth1ApiBinding.java:70) org.springframework.social.twitter.api.impl.TwitterTemplate.(TwitterTemplate.java:79) sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57) sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) java.lang.reflect.Constructor.newInstance(Constructor.java:526) org.springframework.beans.BeanUtils.instantiateClass(BeanUtils.java:148)	... 52 more {code}
Story points: 5