Bug: an error occurred while using the udf jars located in the lakefs, a inner filesystem in Tencent Cloud DLC.
Status: In Progress | Priority: Major | Resolution: 
Created: 29/Nov/21 13:12
Affects Version/s: 3.2.0

when using livy to execute sql statements that will call the udf jars located in lakefs, a inner filesystem in Tencent Cloud DLC. it will threw the following exceptions:

 
{code:java}
21/11/25 21:12:43 ERROR Session: Exception when executing code
java.lang.LinkageError: loader constraint violation: loader (instance of sun/misc/Launcher$AppClassLoader) previously initiated loading for a different type with name "com/qcloud/cos/auth/COSCredentials"
	at java.lang.ClassLoader.defineClass1(Native Method)
	at java.lang.ClassLoader.defineClass(ClassLoader.java:756)
	at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:468)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:74)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:369)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:363)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:362)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:418)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:351)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at org.apache.hadoop.conf.Configuration.getClassByNameOrNull(Configuration.java:2306)
	at org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:2271)
	at org.apache.hadoop.conf.Configuration.getClasses(Configuration.java:2344)
	at org.apache.hadoop.fs.CosNUtils.loadCosProviderClasses(CosNUtils.java:68)
	at org.apache.hadoop.fs.CosFileSystem.initRangerClientImpl(CosFileSystem.java:848)
	at org.apache.hadoop.fs.CosFileSystem.initialize(CosFileSystem.java:95)
	at com.tencent.cloud.fs.CompatibleFileSystem.initialize(CompatibleFileSystem.java:20)
	at com.tencent.cloud.fs.LakeFileSystem.initialize(LakeFileSystem.java:56)
	at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2812)
	at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:100)
	at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2849)
	at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2831)
	at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:389)
	at org.apache.hadoop.fs.FsUrlConnection.connect(FsUrlConnection.java:49)
	at org.apache.hadoop.fs.FsUrlConnection.getInputStream(FsUrlConnection.java:59)
	at sun.net.www.protocol.jar.URLJarFile.retrieve(URLJarFile.java:214)
	at sun.net.www.protocol.jar.URLJarFile.getJarFile(URLJarFile.java:71)
	at sun.net.www.protocol.jar.JarFileFactory.get(JarFileFactory.java:84)
	at sun.net.www.protocol.jar.JarURLConnection.connect(JarURLConnection.java:122)
	at sun.net.www.protocol.jar.JarURLConnection.getJarFile(JarURLConnection.java:89)
	at sun.misc.URLClassPath$JarLoader.getJarFile(URLClassPath.java:944)
	at sun.misc.URLClassPath$JarLoader.access$800(URLClassPath.java:801)
	at sun.misc.URLClassPath$JarLoader$1.run(URLClassPath.java:886)
	at sun.misc.URLClassPath$JarLoader$1.run(URLClassPath.java:879)
	at java.security.AccessController.doPrivileged(Native Method)
	at sun.misc.URLClassPath$JarLoader.ensureOpen(URLClassPath.java:878)
	at sun.misc.URLClassPath$JarLoader.<init>(URLClassPath.java:829)
	at sun.misc.URLClassPath$3.run(URLClassPath.java:575)
	at sun.misc.URLClassPath$3.run(URLClassPath.java:565)
	at java.security.AccessController.doPrivileged(Native Method)
	at sun.misc.URLClassPath.getLoader(URLClassPath.java:564)
	at sun.misc.URLClassPath.getLoader(URLClassPath.java:529)
	at sun.misc.URLClassPath.getNextLoader(URLClassPath.java:494)
	at sun.misc.URLClassPath.access$100(URLClassPath.java:66)
	at sun.misc.URLClassPath$1.next(URLClassPath.java:276)
	at sun.misc.URLClassPath$1.hasMoreElements(URLClassPath.java:287)
	at java.net.URLClassLoader$3$1.run(URLClassLoader.java:604)
	at java.net.URLClassLoader$3$1.run(URLClassLoader.java:602)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader$3.next(URLClassLoader.java:601)
	at java.net.URLClassLoader$3.hasMoreElements(URLClassLoader.java:626)
	at sun.misc.CompoundEnumeration.next(CompoundEnumeration.java:45)
	at sun.misc.CompoundEnumeration.hasMoreElements(CompoundEnumeration.java:54)
	at java.util.ServiceLoader$LazyIterator.hasNextService(ServiceLoader.java:354)
	at java.util.ServiceLoader$LazyIterator.hasNext(ServiceLoader.java:393)
	at java.util.ServiceLoader$1.hasNext(ServiceLoader.java:474)
	at javax.xml.stream.FactoryFinder$1.run(FactoryFinder.java:352)
	at java.security.AccessController.doPrivileged(Native Method)
	at javax.xml.stream.FactoryFinder.findServiceProvider(FactoryFinder.java:341)
	at javax.xml.stream.FactoryFinder.find(FactoryFinder.java:313)
	at javax.xml.stream.FactoryFinder.find(FactoryFinder.java:227)
	at javax.xml.stream.XMLInputFactory.newInstance(XMLInputFactory.java:154)
	at com.qcloud.cos.internal.CosErrorResponseHandler.<clinit>(CosErrorResponseHandler.java:49)
	at com.qcloud.cos.http.DefaultCosHttpClient.<init>(DefaultCosHttpClient.java:92)
	at com.qcloud.cos.COSClient.<init>(COSClient.java:277)
	at org.apache.hadoop.fs.CosNativeFileSystemStore.initCOSClient(CosNativeFileSystemStore.java:188)
	at org.apache.hadoop.fs.CosNativeFileSystemStore.initialize(CosNativeFileSystemStore.java:194)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:409)
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:163)
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:155)
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:95)
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:346)
	at com.sun.proxy.$Proxy38.initialize(Unknown Source)
	at org.apache.hadoop.fs.CosFileSystem.initialize(CosFileSystem.java:103)
	at com.tencent.cloud.fs.CompatibleFileSystem.initialize(CompatibleFileSystem.java:20)
	at com.tencent.cloud.fs.LakeFileSystem.initialize(LakeFileSystem.java:56)
	at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2812)
	at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:100)
	at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2849)
	at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2831)
	at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:389)
	at org.apache.hadoop.hive.ql.util.ResourceDownloader.downloadResource(ResourceDownloader.java:103)
	at org.apache.hadoop.hive.ql.util.ResourceDownloader.resolveAndDownloadInternal(ResourceDownloader.java:88)
	at org.apache.hadoop.hive.ql.util.ResourceDownloader.resolveAndDownload(ResourceDownloader.java:74)
	at org.apache.hadoop.hive.ql.session.SessionState.resolveAndDownload(SessionState.java:1398)
	at org.apache.hadoop.hive.ql.session.SessionState.add_resources(SessionState.java:1350)
	at org.apache.hadoop.hive.ql.session.SessionState.add_resources(SessionState.java:1336)
	at org.apache.hadoop.hive.ql.processors.AddResourceProcessor.run(AddResourceProcessor.java:74)
	at org.apache.spark.sql.hive.client.HiveClientImpl.$anonfun$runHive$1(HiveClientImpl.scala:874)
	at org.apache.spark.sql.hive.client.HiveClientImpl.$anonfun$withHiveState$1(HiveClientImpl.scala:312)
	at org.apache.spark.sql.hive.client.HiveClientImpl.liftedTree1$1(HiveClientImpl.scala:229)
	at org.apache.spark.sql.hive.client.HiveClientImpl.retryLocked(HiveClientImpl.scala:228)
	at org.apache.spark.sql.hive.client.HiveClientImpl.withHiveState(HiveClientImpl.scala:292)
	at org.apache.spark.sql.hive.client.HiveClientImpl.runHive(HiveClientImpl.scala:836)
	at org.apache.spark.sql.hive.client.HiveClientImpl.runSqlHive(HiveClientImpl.scala:823)
	at org.apache.spark.sql.hive.client.HiveClientImpl.addJar(HiveClientImpl.scala:983)
	at org.apache.spark.sql.hive.HiveSessionResourceLoader.addJar(HiveSessionStateBuilder.scala:127)
	at org.apache.spark.sql.internal.SessionResourceLoader.loadResource(SessionState.scala:151)
	at org.apache.spark.sql.catalyst.catalog.SessionCatalog.$anonfun$loadFunctionResources$1(SessionCatalog.scala:1325)
	at org.apache.spark.sql.catalyst.catalog.SessionCatalog.$anonfun$loadFunctionResources$1$adapted(SessionCatalog.scala:1325)
	at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)
	at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)
	at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)
	at org.apache.spark.sql.catalyst.catalog.SessionCatalog.loadFunctionResources(SessionCatalog.scala:1325)
	at org.apache.spark.sql.catalyst.catalog.SessionCatalog.lookupFunction(SessionCatalog.scala:1470)
	at org.apache.spark.sql.hive.HiveSessionCatalog.super$lookupFunction(HiveSessionCatalog.scala:135)
	at org.apache.spark.sql.hive.HiveSessionCatalog.$anonfun$lookupFunction0$2(HiveSessionCatalog.scala:135)
	at scala.util.Try$.apply(Try.scala:213)
	at org.apache.spark.sql.hive.HiveSessionCatalog.lookupFunction0(HiveSessionCatalog.scala:135)
	at org.apache.spark.sql.hive.HiveSessionCatalog.lookupFunction(HiveSessionCatalog.scala:121)
	at org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveFunctions$$anonfun$apply$16$$anonfun$applyOrElse$103.$anonfun$applyOrElse$106(Analyzer.scala:1852)
	at org.apache.spark.sql.catalyst.analysis.package$.withPosition(package.scala:53)
	at org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveFunctions$$anonfun$apply$16$$anonfun$applyOrElse$103.applyOrElse(Analyzer.scala:1852)
	at org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveFunctions$$anonfun$apply$16$$anonfun$applyOrElse$103.applyOrElse(Analyzer.scala:1835)
	at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDown$1(TreeNode.scala:315)
	at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:73)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:315)
	at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDown$3(TreeNode.scala:320)
	at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$mapChildren$1(TreeNode.scala:405)
	at org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:243)
	at org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren(TreeNode.scala:403)
	at org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren(TreeNode.scala:356)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:320)
	at org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$transformExpressionsDown$1(QueryPlan.scala:98)
	at org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$mapExpressions$1(QueryPlan.scala:120)
	at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:73)
	at org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpression$1(QueryPlan.scala:120)
	at org.apache.spark.sql.catalyst.plans.QueryPlan.recursiveTransform$1(QueryPlan.scala:131)
	at org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$mapExpressions$3(QueryPlan.scala:136)
	at scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:238)
	at scala.collection.immutable.List.foreach(List.scala:392)
	at scala.collection.TraversableLike.map(TraversableLike.scala:238)
	at scala.collection.TraversableLike.map$(TraversableLike.scala:231)
	at scala.collection.immutable.List.map(List.scala:298)
	at org.apache.spark.sql.catalyst.plans.QueryPlan.recursiveTransform$1(QueryPlan.scala:136)
	at org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$mapExpressions$4(QueryPlan.scala:141)
	at org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:243)
	at org.apache.spark.sql.catalyst.plans.QueryPlan.mapExpressions(QueryPlan.scala:141)
	at org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpressionsDown(QueryPlan.scala:98)
	at org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpressions(QueryPlan.scala:89)
	at org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveFunctions$$anonfun$apply$16.applyOrElse(Analyzer.scala:1835)
	at org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveFunctions$$anonfun$apply$16.applyOrElse(Analyzer.scala:1833)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.$anonfun$resolveOperatorsUp$3(AnalysisHelper.scala:90)
	at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:73)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.$anonfun$resolveOperatorsUp$1(AnalysisHelper.scala:90)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper$.allowInvokingTransformsInAnalyzer(AnalysisHelper.scala:212)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.resolveOperatorsUp(AnalysisHelper.scala:86)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.resolveOperatorsUp$(AnalysisHelper.scala:84)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.resolveOperatorsUp(LogicalPlan.scala:29)
	at org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveFunctions$.apply(Analyzer.scala:1833)
	at org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveFunctions$.apply(Analyzer.scala:1831)
	at org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$2(RuleExecutor.scala:149)
	at scala.collection.LinearSeqOptimized.foldLeft(LinearSeqOptimized.scala:126)
	at scala.collection.LinearSeqOptimized.foldLeft$(LinearSeqOptimized.scala:122)
	at scala.collection.immutable.List.foldLeft(List.scala:89)
	at org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$1(RuleExecutor.scala:146)
	at org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$1$adapted(RuleExecutor.scala:138)
	at scala.collection.immutable.List.foreach(List.scala:392)
	at org.apache.spark.sql.catalyst.rules.RuleExecutor.execute(RuleExecutor.scala:138)
	at org.apache.spark.sql.catalyst.analysis.Analyzer.org$apache$spark$sql$catalyst$analysis$Analyzer$$executeSameContext(Analyzer.scala:171)
	at org.apache.spark.sql.catalyst.analysis.Analyzer.execute(Analyzer.scala:165)
	at org.apache.spark.sql.catalyst.analysis.Analyzer.execute(Analyzer.scala:130)
	at org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$executeAndTrack$1(RuleExecutor.scala:116)
	at org.apache.spark.sql.catalyst.QueryPlanningTracker$.withTracker(QueryPlanningTracker.scala:88)
	at org.apache.spark.sql.catalyst.rules.RuleExecutor.executeAndTrack(RuleExecutor.scala:116)
	at org.apache.spark.sql.catalyst.analysis.Analyzer.$anonfun$executeAndCheck$1(Analyzer.scala:149)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper$.markInAnalyzer(AnalysisHelper.scala:219)
	at org.apache.spark.sql.catalyst.analysis.Analyzer.executeAndCheck(Analyzer.scala:148)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$analyzed$1(QueryExecution.scala:68)
	at org.apache.spark.sql.catalyst.QueryPlanningTracker.measurePhase(QueryPlanningTracker.scala:111)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$executePhase$1(QueryExecution.scala:138)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:771)
	at org.apache.spark.sql.execution.QueryExecution.executePhase(QueryExecution.scala:138)
	at org.apache.spark.sql.execution.QueryExecution.analyzed$lzycompute(QueryExecution.scala:68)
	at org.apache.spark.sql.execution.QueryExecution.analyzed(QueryExecution.scala:66)
	at org.apache.spark.sql.execution.QueryExecution.assertAnalyzed(QueryExecution.scala:58)
	at org.apache.spark.sql.Dataset$.$anonfun$ofRows$2(Dataset.scala:99)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:771)
	at org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:97)
	at org.apache.spark.sql.SparkSession.$anonfun$sql$1(SparkSession.scala:614)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:771)
	at org.apache.spark.sql.SparkSession.sql(SparkSession.scala:609)
	at org.apache.livy.repl.SQLInterpreter.execute(SQLInterpreter.scala:93)
	at org.apache.livy.repl.Session.$anonfun$executeCode$1(Session.scala:330)
	at scala.Option.map(Option.scala:230)
	at org.apache.livy.repl.Session.executeCode(Session.scala:328)
	at org.apache.livy.repl.Session.$anonfun$execute$1(Session.scala:175)
	at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
	at scala.concurrent.Future$.$anonfun$apply$1(Future.scala:659)
	at scala.util.Success.$anonfun$map$1(Try.scala:255)
	at scala.util.Success.map(Try.scala:213)
	at scala.concurrent.Future.$anonfun$map$1(Future.scala:292)
	at scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33)
	at scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33)
	at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748) {code}
 

i add `-XX:+TraceLoaderConstraints` in `spark.driver.extraJavaOptions` which is used to trace loader constraint，and then we can see the bug that different class loaders load the same class by the attachment。