Bug:  Spark set configuration parameters take a long time
Status: Open | Priority: Major | Resolution: 
Created: 21/Nov/23 08:17
Affects Version/s: 3.3.1

set spark.sql.sources.partitionOverwriteMode = dynamic;
SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
SLF4J: Defaulting to no-operation (NOP) logger implementation
SLF4J: See [http://www.slf4j.org/codes.html#StaticLoggerBinder] for further details.
spark.sql.sources.partitionOverwriteMode        dynamic
Time taken: 3.498 seconds, Fetched 1 row(s)

 

`---ts=2023-11-21 15:01:22;thread_name=main;id=1;is_daemon=false;priority=5;TCCL=org.apache.spark.util.MutableURLClassLoader@5f2bd6d9
    `---[3076.713713ms] org.apache.spark.sql.SparkSession:withActive()
        +---[0.00% 0.021457ms ] org.apache.spark.sql.SparkSession$:org$apache$spark$sql$SparkSession$$activeThreadSession() #804
        +---[0.00% 0.02545ms ] org.apache.spark.sql.SparkSession$:setActiveSession() #805
        +---[99.99% 3076.539159ms ] scala.Function0:apply() #806
        |   `---[89.78% 2761.986102ms ] org.apache.spark.sql.SparkSession:withActive()
        |       +---[0.00% 0.009825ms ] org.apache.spark.sql.SparkSession$:org$apache$spark$sql$SparkSession$$activeThreadSession() #804
        |       +---[0.00% 0.009074ms ] org.apache.spark.sql.SparkSession$:setActiveSession() #805
        |       +---[100.00% 2761.884144ms ] scala.Function0:apply() #806
        |       |   `---[47.70% min=469.98645ms,max=847.488281ms,total=1317.474731ms,count=2] org.apache.spark.sql.SparkSession:withActive()
        |       |       +---[0.00% min=0.008888ms,max=0.009888ms,total=0.018776ms,count=2] org.apache.spark.sql.SparkSession$:org$apache$spark$sql$SparkSession$$activeThreadSession() #804
        |       |       +---[0.00% min=0.00653ms,max=0.010794ms,total=0.017324ms,count=2] org.apache.spark.sql.SparkSession$:setActiveSession() #805
        |       |       +---[99.98% min=469.884777ms,max=847.357657ms,total=1317.242434ms,count=2] scala.Function0:apply() #806
        |       |       |   `---[31.81% min=0.121928ms,max=291.731882ms,total=419.046782ms,count=3] org.apache.spark.sql.SparkSession:withActive()
        |       |       |       +---[0.01% min=0.005732ms,max=0.029045ms,total=0.041459ms,count=3] org.apache.spark.sql.SparkSession$:org$apache$spark$sql$SparkSession$$activeThreadSession() #804
        |       |       |       +---[0.00% min=0.005249ms,max=0.008029ms,total=0.020456ms,count=3] org.apache.spark.sql.SparkSession$:setActiveSession() #805
        |       |       |       +---[99.94% min=0.02566ms,max=291.640405ms,total=418.792745ms,count=3] scala.Function0:apply() #806
        |       |       |       |   `---[15.36% min=18.869542ms,max=45.477222ms,total=64.346764ms,count=2] org.apache.spark.sql.SparkSession:withActive()
        |       |       |       |       +---[0.02% min=0.006741ms,max=0.007684ms,total=0.014425ms,count=2] org.apache.spark.sql.SparkSession$:org$apache$spark$sql$SparkSession$$activeThreadSession() #804
        |       |       |       |       +---[0.02% min=0.005835ms,max=0.006769ms,total=0.012604ms,count=2] org.apache.spark.sql.SparkSession$:setActiveSession() #805
        |       |       |       |       +---[99.75% min=18.78687ms,max=45.397814ms,total=64.184684ms,count=2] scala.Function0:apply() #806
        |       |       |       |       `---[0.02% min=0.007134ms,max=0.008043ms,total=0.015177ms,count=2] org.apache.spark.sql.SparkSession$:setActiveSession() #807
        |       |       |       `---[0.00% min=0.005495ms,max=0.007487ms,total=0.020001ms,count=3] org.apache.spark.sql.SparkSession$:setActiveSession() #807
        |       |       `---[0.00% min=0.010495ms,max=0.010719ms,total=0.021214ms,count=2] org.apache.spark.sql.SparkSession$:setActiveSession() #807
        |       `---[0.00% 0.010261ms ] org.apache.spark.sql.SparkSession$:setActiveSession() #807
        `---[0.00% 0.008061ms ] org.apache.spark.sql.SparkSession$:setActiveSession() #807

`---ts=2023-11-21 15:01:25;thread_name=main;id=1;is_daemon=false;priority=5;TCCL=org.apache.spark.util.MutableURLClassLoader@5f2bd6d9
    `---[389.831588ms] org.apache.spark.sql.SparkSession:withActive()
        +---[0.00% 0.009996ms ] org.apache.spark.sql.SparkSession$:org$apache$spark$sql$SparkSession$$activeThreadSession() #804
        +---[0.00% 0.008191ms ] org.apache.spark.sql.SparkSession$:setActiveSession() #805
        +---[99.98% 389.738962ms ] scala.Function0:apply() #806
        |   `---[15.01% min=10.215178ms,max=32.7175ms,total=58.512671ms,count=3] org.apache.spark.sql.SparkSession:withActive()
        |       +---[0.03% min=0.00496ms,max=0.009591ms,total=0.019529ms,count=3] org.apache.spark.sql.SparkSession$:org$apache$spark$sql$SparkSession$$activeThreadSession() #804
        |       +---[0.03% min=0.004656ms,max=0.008853ms,total=0.0182ms,count=3] org.apache.spark.sql.SparkSession$:setActiveSession() #805
        |       +---[99.64% min=10.153047ms,max=32.624937ms,total=58.302978ms,count=3] scala.Function0:apply() #806
        |       |   `---[26.07% min=1.042219ms,max=14.160052ms,total=15.202271ms,count=2] org.apache.spark.sql.SparkSession:withActive()
        |       |       +---[0.06% min=0.004344ms,max=0.005206ms,total=0.00955ms,count=2] org.apache.spark.sql.SparkSession$:org$apache$spark$sql$SparkSession$$activeThreadSession() #804
        |       |       +---[0.06% min=0.004451ms,max=0.004847ms,total=0.009298ms,count=2] org.apache.spark.sql.SparkSession$:setActiveSession() #805
        |       |       +---[99.23% min=0.989634ms,max=14.096249ms,total=15.085883ms,count=2] scala.Function0:apply() #806
        |       |       `---[0.09% min=0.006822ms,max=0.007377ms,total=0.014199ms,count=2] org.apache.spark.sql.SparkSession$:setActiveSession() #807
        |       `---[0.03% min=0.006345ms,max=0.007115ms,total=0.020436ms,count=3] org.apache.spark.sql.SparkSession$:setActiveSession() #807
        `---[0.00% 0.008346ms ] org.apache.spark.sql.SparkSession$:setActiveSession() #807