Bug: AQEPropagateEmptyRelation convert broadcast query stage plan to empty relation causing error
Status: Open | Priority: Major | Resolution: 
Created: 28/Jun/24 09:04
Affects Version/s: 3.2.4

{code:java}
    at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
Caused by: java.util.concurrent.ExecutionException: Boxed Error
    at scala.concurrent.impl.Promise$.resolver(Promise.scala:87)
    at scala.concurrent.impl.Promise$.scala$concurrent$impl$Promise$$resolveTry(Promise.scala:79)
    at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:284)
    at scala.concurrent.Promise.complete(Promise.scala:53)
    at scala.concurrent.Promise.complete$(Promise.scala:52)
    at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:187)
    at scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33)
    at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
    at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.AssertionError: assertion failed
    at scala.Predef$.assert(Predef.scala:208)
    at org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec.$anonfun$doExecuteBroadcast$1(AdaptiveSparkPlanExec.scala:369)
    at org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec.withFinalPlanUpdate(AdaptiveSparkPlanExec.scala:376)
    at org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec.doExecuteBroadcast(AdaptiveSparkPlanExec.scala:364)
    at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeBroadcast$1(SparkPlan.scala:197)
    at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:222)
    at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
    at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:219)
    at org.apache.spark.sql.execution.SparkPlan.executeBroadcast(SparkPlan.scala:193)
    at org.apache.spark.sql.execution.SubqueryBroadcastExec.$anonfun$relationFuture$2(SubqueryBroadcastExec.scala:81)
    at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withExecutionId$1(SQLExecution.scala:146)
    at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:170)
    at org.apache.spark.sql.execution.SQLExecution$.withExecutionId(SQLExecution.scala:144)
    at org.apache.spark.sql.execution.SubqueryBroadcastExec.$anonfun$relationFuture$1(SubqueryBroadcastExec.scala:78)
    at scala.concurrent.Future$.$anonfun$apply$1(Future.scala:659)
    at scala.util.Success.$anonfun$map$1(Try.scala:255)
    at scala.util.Success.map(Try.scala:213)
    at scala.concurrent.Future.$anonfun$map$1(Future.scala:292)
    at scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33)
    ... 5 more
 {code}
 

 
{code:java}
=== Applying Rule org.apache.spark.sql.execution.adaptive.AQEPropagateEmptyRelation ===
 Project [bm_id#1469, uid#1491L, ... 4 more fields]                                                                                                                                                                                         Project [bm_id#1469, uid#1491L, ... 4 more fields]
!+- Join Inner, (uid#1491L = uid#1495L)                                                                                                                                                                                                     +- LocalRelation <empty>, [bm_id#1469, uid#1491L, ... 5 more fields]
!   :- LogicalQueryStage Project [xx#1469, uid#1491L, ... 4 more fields], BroadcastQueryStage 1
!   +- Project [uid#1495L]
!      +- Filter xxxx
!         +- Relation db.table[uid#1495L,request_id#1496,... 40 more fields] parquet {code}