Bug: flaky test in pyspark-core test
Status: Open | Priority: Major | Resolution: 
Created: 27/Nov/24 06:56
Affects Version/s: 4.0.0

# [https://github.com/apache/spark/actions/runs/12044265847/job/33581052605]
 # [https://github.com/apache/spark/actions/runs/12043870433/job/33579994907]
 # [https://github.com/LuciferYang/spark/actions/runs/12034481725/job/33551257684]
 # [https://github.com/LuciferYang/spark/actions/runs/12034633745/job/33554178490]

{code:java}
Test the resources are available. ... java.util.concurrent.RejectedExecutionException: Task Future(<not completed>) rejected from java.util.concurrent.ThreadPoolExecutor@38ae8ffe[Shutting down, pool size = 1, active threads = 1, queued tasks = 0, completed tasks = 0]
3835	at java.base/java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065)
3836	at java.base/java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833)
3837	at java.base/java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365)
3838	at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21)
3839	at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429)
3840	at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338)
3841	at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285)
3842	at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504)
3843	at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222)
3844	at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429)
3845	at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338)
3846	at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285)
3847	at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504)
3848	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
3849	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
3850	at java.base/java.lang.Thread.run(Thread.java:840) {code}
{code:java}
py4j.protocol.Py4JJavaError: An error occurred while calling o20.setLogLevel.
4768: java.util.ConcurrentModificationException
4769	at java.base/java.util.HashMap$ValueSpliterator.forEachRemaining(HashMap.java:1784)
4770	at java.base/java.util.stream.ReferencePipeline$Head.forEach(ReferencePipeline.java:762)
4771	at java.base/java.util.stream.ReferencePipeline$7$1.accept(ReferencePipeline.java:276)
4772	at java.base/java.util.WeakHashMap$ValueSpliterator.forEachRemaining(WeakHashMap.java:1217)
4773	at java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:509)
4774	at java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:499)
4775	at java.base/java.util.stream.ForEachOps$ForEachOp.evaluateSequential(ForEachOps.java:150)
4776	at java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(ForEachOps.java:173)
4777	at java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
4778	at java.base/java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:596)
4779	at org.apache.logging.log4j.core.LoggerContext.updateLoggers(LoggerContext.java:776)
4780	at org.apache.logging.log4j.core.LoggerContext.updateLoggers(LoggerContext.java:766)
4781	at org.apache.spark.util.Utils$.setLogLevel(Utils.scala:2322)
4782	at org.apache.spark.util.Utils$.setLogLevelIfNeeded(Utils.scala:2331)
4783	at org.apache.spark.SparkContext.setLogLevel(SparkContext.scala:400)
4784	at org.apache.spark.api.java.JavaSparkContext.setLogLevel(JavaSparkContext.scala:675)
4785	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
4786	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
4787	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
4788	at java.base/java.lang.reflect.Method.invoke(Method.java:569)
4789	at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
4790	at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)
4791	at py4j.Gateway.invoke(Gateway.java:282)
4792	at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
4793	at py4j.commands.CallCommand.execute(CallCommand.java:79)
4794	at py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
4795	at py4j.ClientServerConnection.run(ClientServerConnection.java:106)
4796	at java.base/java.lang.Thread.run(Thread.java:840) {code}