Bug: NullPointerException in UnwrapCastInBinaryComparison
Status: Open | Priority: Major | Resolution: 
Created: 03/Sep/24 09:38
Affects Version/s: 3.5.3

How to reproduce:
{code:scala}
spark.sql("CREATE TABLE fact_cc_summary_all_mcc (trans_cnt_3ds BIGINT) USING parquet")
spark.sql("SELECT cast(trans_cnt_3ds as decimal (8,4))<0=.5 FROM fact_cc_summary_all_mcc as mstr").show
{code}


{noformat}
scala> spark.sql("SELECT cast(trans_cnt_3ds as decimal (8,4))<0=.5 FROM fact_cc_summary_all_mcc as mstr").show
org.apache.spark.SparkException: [INTERNAL_ERROR] The Spark SQL phase optimization failed with an internal error. You hit a bug in Spark or the Spark plugins you use. Please, report this bug to the corresponding communities or vendors, and provide the full stack trace.
  at org.apache.spark.SparkException$.internalError(SparkException.scala:107)
  at org.apache.spark.sql.execution.QueryExecution$.toInternalError(QueryExecution.scala:536)
  at org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:548)
  at org.apache.spark.sql.execution.QueryExecution.$anonfun$executePhase$1(QueryExecution.scala:219)
  at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:900)
  at org.apache.spark.sql.execution.QueryExecution.executePhase(QueryExecution.scala:218)
  at org.apache.spark.sql.execution.QueryExecution.optimizedPlan$lzycompute(QueryExecution.scala:148)
  at org.apache.spark.sql.execution.QueryExecution.optimizedPlan(QueryExecution.scala:144)
  at org.apache.spark.sql.execution.QueryExecution.assertOptimized(QueryExecution.scala:162)
  at org.apache.spark.sql.execution.QueryExecution.executedPlan$lzycompute(QueryExecution.scala:182)
  at org.apache.spark.sql.execution.QueryExecution.executedPlan(QueryExecution.scala:179)
  at org.apache.spark.sql.execution.QueryExecution.simpleString(QueryExecution.scala:238)
  at org.apache.spark.sql.execution.QueryExecution.org$apache$spark$sql$execution$QueryExecution$$explainString(QueryExecution.scala:284)
  at org.apache.spark.sql.execution.QueryExecution.explainString(QueryExecution.scala:252)
  at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$6(SQLExecution.scala:117)
  at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:201)
  at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:108)
  at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:900)
  at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:66)
  at org.apache.spark.sql.Dataset.withAction(Dataset.scala:4322)
  at org.apache.spark.sql.Dataset.head(Dataset.scala:3316)
  at org.apache.spark.sql.Dataset.take(Dataset.scala:3539)
  at org.apache.spark.sql.Dataset.getRows(Dataset.scala:280)
  at org.apache.spark.sql.Dataset.showString(Dataset.scala:315)
  at org.apache.spark.sql.Dataset.show(Dataset.scala:838)
  at org.apache.spark.sql.Dataset.show(Dataset.scala:797)
  at org.apache.spark.sql.Dataset.show(Dataset.scala:806)
  ... 47 elided
Caused by: java.lang.NullPointerException: Cannot invoke "org.apache.spark.sql.types.Decimal.toBigDecimal()" because "other" is null
  at org.apache.spark.sql.types.Decimal.compare(Decimal.scala:460)
  at org.apache.spark.sql.types.Decimal$DecimalIsConflicted.compare(Decimal.scala:687)
  at org.apache.spark.sql.types.Decimal$DecimalIsConflicted.compare$(Decimal.scala:687)
  at org.apache.spark.sql.types.Decimal$DecimalIsFractional$.compare(Decimal.scala:694)
  at org.apache.spark.sql.types.Decimal$DecimalIsFractional$.compare(Decimal.scala:694)
  at org.apache.spark.sql.catalyst.optimizer.UnwrapCastInBinaryComparison$.simplifyNumericComparison(UnwrapCastInBinaryComparison.scala:200)
  at org.apache.spark.sql.catalyst.optimizer.UnwrapCastInBinaryComparison$.org$apache$spark$sql$catalyst$optimizer$UnwrapCastInBinaryComparison$$unwrapCast(UnwrapCastInBinaryComparison.scala:134)
  at org.apache.spark.sql.catalyst.optimizer.UnwrapCastInBinaryComparison$$anonfun$apply$2$$anonfun$applyOrElse$2.applyOrElse(UnwrapCastInBinaryComparison.scala:108)
  at org.apache.spark.sql.catalyst.optimizer.UnwrapCastInBinaryComparison$$anonfun$apply$2$$anonfun$applyOrElse$2.applyOrElse(UnwrapCastInBinaryComparison.scala:107)
{noformat}