Bug: Few ITs (using MiniMRYarnCluster on hadoop-2) are failing due to NCDFE: com/sun/jersey/core/util/FeaturesAndProperties
Status: Resolved | Priority: Major | Resolution: Fixed
Created: 14/Jan/24 06:57

Found this while trying to run tests for HBASE-28301 locally, On branch-2 where hadoop 2 is default, the specified tests don't even run as MiniMRYarnCluster itself fails to start.

For example saw this while trying to run IntegrationTestImportTsv:
{code:java}
2024-01-12T01:10:13,486 ERROR [Thread-221 {}] log.Slf4jLog(87): Error starting handlers 
java.lang.NoClassDefFoundError: com/sun/jersey/core/util/FeaturesAndProperties
    at java.lang.ClassLoader.defineClass1(Native Method) ~[?:1.8.0_381]
    at java.lang.ClassLoader.defineClass(ClassLoader.java:756) ~[?:1.8.0_381]
    at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142) ~[?:1.8.0_381]
    at java.net.URLClassLoader.defineClass(URLClassLoader.java:473) ~[?:1.8.0_381]
    at java.net.URLClassLoader.access$100(URLClassLoader.java:74) ~[?:1.8.0_381]
    at java.net.URLClassLoader$1.run(URLClassLoader.java:369) ~[?:1.8.0_381]
    at java.net.URLClassLoader$1.run(URLClassLoader.java:363) ~[?:1.8.0_381]
    at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_381]
    at java.net.URLClassLoader.findClass(URLClassLoader.java:362) ~[?:1.8.0_381]
    at java.lang.ClassLoader.loadClass(ClassLoader.java:418) ~[?:1.8.0_381]
    at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:355) ~[?:1.8.0_381]
    at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ~[?:1.8.0_381]
    at java.lang.ClassLoader.defineClass1(Native Method) ~[?:1.8.0_381]
    at java.lang.ClassLoader.defineClass(ClassLoader.java:756) ~[?:1.8.0_381]
    at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142) ~[?:1.8.0_381]
    at java.net.URLClassLoader.defineClass(URLClassLoader.java:473) ~[?:1.8.0_381]
    at java.net.URLClassLoader.access$100(URLClassLoader.java:74) ~[?:1.8.0_381]
    at java.net.URLClassLoader$1.run(URLClassLoader.java:369) ~[?:1.8.0_381]
    at java.net.URLClassLoader$1.run(URLClassLoader.java:363) ~[?:1.8.0_381]
    at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_381]
    at java.net.URLClassLoader.findClass(URLClassLoader.java:362) ~[?:1.8.0_381]
    at java.lang.ClassLoader.loadClass(ClassLoader.java:418) ~[?:1.8.0_381]
    at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:355) ~[?:1.8.0_381]
    at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ~[?:1.8.0_381]
    at java.lang.ClassLoader.defineClass1(Native Method) ~[?:1.8.0_381]
    at java.lang.ClassLoader.defineClass(ClassLoader.java:756) ~[?:1.8.0_381]
    at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142) ~[?:1.8.0_381]
    at java.net.URLClassLoader.defineClass(URLClassLoader.java:473) ~[?:1.8.0_381]
    at java.net.URLClassLoader.access$100(URLClassLoader.java:74) ~[?:1.8.0_381]
    at java.net.URLClassLoader$1.run(URLClassLoader.java:369) ~[?:1.8.0_381]
    at java.net.URLClassLoader$1.run(URLClassLoader.java:363) ~[?:1.8.0_381]
    at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_381]
    at java.net.URLClassLoader.findClass(URLClassLoader.java:362) ~[?:1.8.0_381]
    at java.lang.ClassLoader.loadClass(ClassLoader.java:418) ~[?:1.8.0_381]
    at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:355) ~[?:1.8.0_381]
    at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ~[?:1.8.0_381]
    at java.lang.Class.getDeclaredConstructors0(Native Method) ~[?:1.8.0_381]
    at java.lang.Class.privateGetDeclaredConstructors(Class.java:2671) ~[?:1.8.0_381]
    at java.lang.Class.getDeclaredConstructors(Class.java:2020) ~[?:1.8.0_381]
    at com.google.inject.spi.InjectionPoint.forConstructorOf(InjectionPoint.java:243) ~[guice-3.0.jar:?]
    at com.google.inject.internal.ConstructorBindingImpl.create(ConstructorBindingImpl.java:96) ~[guice-3.0.jar:?]
    at com.google.inject.internal.InjectorImpl.createUninitializedBinding(InjectorImpl.java:629) ~[guice-3.0.jar:?]
    at com.google.inject.internal.InjectorImpl.createJustInTimeBinding(InjectorImpl.java:845) ~[guice-3.0.jar:?]
    at com.google.inject.internal.InjectorImpl.createJustInTimeBindingRecursive(InjectorImpl.java:772) ~[guice-3.0.jar:?]
    at com.google.inject.internal.InjectorImpl.getJustInTimeBinding(InjectorImpl.java:256) ~[guice-3.0.jar:?]
    at com.google.inject.internal.InjectorImpl.getBindingOrThrow(InjectorImpl.java:205) ~[guice-3.0.jar:?]
    at com.google.inject.internal.InjectorImpl.getBinding(InjectorImpl.java:146) ~[guice-3.0.jar:?]
    at com.google.inject.internal.InjectorImpl.getBinding(InjectorImpl.java:66) ~[guice-3.0.jar:?]
    at com.google.inject.servlet.FilterDefinition.init(FilterDefinition.java:99) ~[guice-servlet-3.0.jar:?]
    at com.google.inject.servlet.ManagedFilterPipeline.initPipeline(ManagedFilterPipeline.java:98) ~[guice-servlet-3.0.jar:?]
    at com.google.inject.servlet.GuiceFilter.init(GuiceFilter.java:172) ~[guice-servlet-3.0.jar:?]
    at org.mortbay.jetty.servlet.FilterHolder.doStart(FilterHolder.java:97) ~[jetty-6.1.26.jar:6.1.26]
    at org.mortbay.component.AbstractLifeCycle.start(AbstractLifeCycle.java:50) ~[jetty-util-6.1.26.jar:6.1.26]
    at org.mortbay.jetty.servlet.ServletHandler.initialize(ServletHandler.java:713) ~[jetty-6.1.26.jar:6.1.26]
    at org.mortbay.jetty.servlet.Context.startContext(Context.java:140) ~[jetty-6.1.26.jar:6.1.26]
    at org.mortbay.jetty.webapp.WebAppContext.startContext(WebAppContext.java:1282) ~[jetty-6.1.26.jar:6.1.26]
    at org.mortbay.jetty.handler.ContextHandler.doStart(ContextHandler.java:518) ~[jetty-6.1.26.jar:6.1.26]
    at org.mortbay.jetty.webapp.WebAppContext.doStart(WebAppContext.java:499) ~[jetty-6.1.26.jar:6.1.26]
    at org.mortbay.component.AbstractLifeCycle.start(AbstractLifeCycle.java:50) ~[jetty-util-6.1.26.jar:6.1.26]
    at org.mortbay.jetty.handler.HandlerCollection.doStart(HandlerCollection.java:152) ~[jetty-6.1.26.jar:6.1.26]
    at org.mortbay.jetty.handler.ContextHandlerCollection.doStart(ContextHandlerCollection.java:156) ~[jetty-6.1.26.jar:6.1.26]
    at org.mortbay.component.AbstractLifeCycle.start(AbstractLifeCycle.java:50) ~[jetty-util-6.1.26.jar:6.1.26]
    at org.mortbay.jetty.handler.HandlerWrapper.doStart(HandlerWrapper.java:130) ~[jetty-6.1.26.jar:6.1.26]
    at org.mortbay.jetty.Server.doStart(Server.java:224) ~[jetty-6.1.26.jar:6.1.26]
    at org.mortbay.component.AbstractLifeCycle.start(AbstractLifeCycle.java:50) ~[jetty-util-6.1.26.jar:6.1.26]
    at org.apache.hadoop.http.HttpServer2.start(HttpServer2.java:964) ~[hadoop-common-2.10.2.jar:?]
    at org.apache.hadoop.yarn.webapp.WebApps$Builder.start(WebApps.java:408) ~[hadoop-yarn-common-2.10.2.jar:?]
    at org.apache.hadoop.yarn.webapp.WebApps$Builder.start(WebApps.java:397) ~[hadoop-yarn-common-2.10.2.jar:?]
    at org.apache.hadoop.mapreduce.v2.hs.HistoryClientService.initializeWebApp(HistoryClientService.java:156) ~[hadoop-mapreduce-client-hs-2.10.2.jar:?]
    at org.apache.hadoop.mapreduce.v2.hs.HistoryClientService.serviceStart(HistoryClientService.java:121) ~[hadoop-mapreduce-client-hs-2.10.2.jar:?]
    at org.apache.hadoop.service.AbstractService.start(AbstractService.java:194) ~[hadoop-common-2.10.2.jar:?]
    at org.apache.hadoop.service.CompositeService.serviceStart(CompositeService.java:121) ~[hadoop-common-2.10.2.jar:?]
    at org.apache.hadoop.mapreduce.v2.hs.JobHistoryServer.serviceStart(JobHistoryServer.java:202) ~[hadoop-mapreduce-client-hs-2.10.2.jar:?]
    at org.apache.hadoop.service.AbstractService.start(AbstractService.java:194) ~[hadoop-common-2.10.2.jar:?]
    at org.apache.hadoop.mapreduce.v2.MiniMRYarnCluster$JobHistoryServerWrapper$1.run(MiniMRYarnCluster.java:241) ~[hadoop-mapreduce-client-jobclient-2.10.2-tests.jar:?]
Caused by: java.lang.ClassNotFoundException: com.sun.jersey.core.util.FeaturesAndProperties
    at java.net.URLClassLoader.findClass(URLClassLoader.java:387) ~[?:1.8.0_381]
    at java.lang.ClassLoader.loadClass(ClassLoader.java:418) ~[?:1.8.0_381]
    at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:355) ~[?:1.8.0_381]
    at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ~[?:1.8.0_381]
    ... 75 more
2024-01-12T01:10:13,488 INFO  [Thread-221 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@WM-C02FW1LNMD6T:50665
2024-01-12T01:10:13,493 INFO  [Thread-221 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@WM-C02FW1LNMD6T:0
2024-01-12T01:10:13,600 ERROR [Thread[Thread-222,5,main] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
2024-01-12T01:10:13,601 ERROR [Thread-221 {}] server.NIOServerCnxnFactory(85): Thread Thread[Thread-221,5,main] died
org.apache.hadoop.yarn.webapp.WebAppException: Error starting http server
    at org.apache.hadoop.yarn.webapp.WebApps$Builder.start(WebApps.java:412) ~[hadoop-yarn-common-2.10.2.jar:?]
    at org.apache.hadoop.yarn.webapp.WebApps$Builder.start(WebApps.java:397) ~[hadoop-yarn-common-2.10.2.jar:?]
    at org.apache.hadoop.mapreduce.v2.hs.HistoryClientService.initializeWebApp(HistoryClientService.java:156) ~[hadoop-mapreduce-client-hs-2.10.2.jar:?]
    at org.apache.hadoop.mapreduce.v2.hs.HistoryClientService.serviceStart(HistoryClientService.java:121) ~[hadoop-mapreduce-client-hs-2.10.2.jar:?]
    at org.apache.hadoop.service.AbstractService.start(AbstractService.java:194) ~[hadoop-common-2.10.2.jar:?]
    at org.apache.hadoop.service.CompositeService.serviceStart(CompositeService.java:121) ~[hadoop-common-2.10.2.jar:?]
    at org.apache.hadoop.mapreduce.v2.hs.JobHistoryServer.serviceStart(JobHistoryServer.java:202) ~[hadoop-mapreduce-client-hs-2.10.2.jar:?]
    at org.apache.hadoop.service.AbstractService.start(AbstractService.java:194) ~[hadoop-common-2.10.2.jar:?]
    at org.apache.hadoop.mapreduce.v2.MiniMRYarnCluster$JobHistoryServerWrapper$1.run(MiniMRYarnCluster.java:241) ~[hadoop-mapreduce-client-jobclient-2.10.2-tests.jar:?]
Caused by: java.io.IOException: Problem in starting http server. Server handlers failed
    at org.apache.hadoop.http.HttpServer2.start(HttpServer2.java:976) ~[hadoop-common-2.10.2.jar:?]
    at org.apache.hadoop.yarn.webapp.WebApps$Builder.start(WebApps.java:408) ~[hadoop-yarn-common-2.10.2.jar:?]
    ... 8 more{code}
CC: [~zhangduo]