Repository: apache/datafusion (Rust)

Issue:
Comparisons (BinaryExprs) to literal NULL don't work 
**Describe the bug**

Bug is that datafusion SQL treats a literal `NULL` as though it is typed `ScalarValue::Utf8(None)`

So then if you try to compare null to a column that is not a scalar DataFusion goes 💥 

**To Reproduce**
(I *love* the new `VALUES` feature that @Jimexist added)

```sql
> select * from (VALUES (1, 'foo' ,2.3), (2, 'bar', 5.4)) as t;
+---------+---------+---------+
| column1 | column2 | column3 |
+---------+---------+---------+
| 1       | foo     | 2.3     |
| 2       | bar     | 5.4     |
+---------+---------+---------+
2 rows in set. Query took 0.004 seconds.
> select column1 < null from (VALUES (1, 'foo' ,2.3), (2, 'bar', 5.4)) as t;
Plan("'Int64 < Utf8' can't be evaluated because there isn't a common type to coerce the types to")
> select column2 < null from (VALUES (1, 'foo' ,2.3), (2, 'bar', 5.4)) as t;
ArrowError(ExternalError(Internal("compute_utf8_op_scalar for 'lt' failed to cast literal value NULL")))
> select column3 < null from (VALUES (1, 'foo' ,2.3), (2, 'bar', 5.4)) as t;
Plan("'Float64 < Utf8' can't be evaluated because there isn't a common type to coerce the types to")
```

**Expected behavior**
All three queries should return 1 column, 2 rows with all values null;

| result |
| ------|
| NULL |
| NULL |


**Additional context**

This came up in the context of IOx where we are doing partial evaluation (on something that is like at the partition level) where references to columns may be rewritten into `null`s. See more details on  https://github.com/influxdata/influxdb_iox/issues/883



Discussion:
FYI here is what postgres does

```
alamb@MacBook-Pro:/tmp/arrow-rs$ psql
psql (13.4)
Type "help" for help.

alamb=# select * from (VALUES (1, 'foo' ,2.3), (2, 'bar', 5.4)) as t;
 column1 | column2 | column3 
---------+---------+---------
       1 | foo     |     2.3
       2 | bar     |     5.4
(2 rows)

alamb=# select column1 < null from (VALUES (1, 'foo' ,2.3), (2, 'bar', 5.4)) as t;
 ?column? 
----------
 
 
(2 rows)

alamb=# select column2 < null from (VALUES (1, 'foo' ,2.3), (2, 'bar', 5.4)) as t;
 ?column? 
----------
 
 
(2 rows)

alamb=# select column3 < null from (VALUES (1, 'foo' ,2.3), (2, 'bar', 5.4)) as t;
 ?column? 
----------
 
 
(2 rows)

alamb=# 
```
FYI i put `\pset null ¤` into `~/.psqlrc` so that null is explicit

```
[postgres] # select column1 < null from (VALUES (1, 'foo' ,2.3), (2, 'bar', 5.4)) as t;
 ?column?
----------
 ¤
 ¤
(2 rows)
```
The source of the problem is here:
```
                        SQLExpr::Value(Value::Null) => {
                            Ok(Expr::Literal(ScalarValue::Utf8(None)))
                        }
```

I would propose:

* To introduce `ScalarValue::Null` and use that instead for `Value::Null`.
* Return `DataType::Null` as type for `ScalarValue::Null`
* Implement the rest of changes in coercion as needed. 
> The source of the problem is here:
> 
> ```
>                         SQLExpr::Value(Value::Null) => {
>                             Ok(Expr::Literal(ScalarValue::Utf8(None)))
>                         }
> ```
> 
> I would propose:
> 
> * To introduce `ScalarValue::Null` and use that instead for `Value::Null`.
> * Return `DataType::Null` as type for `ScalarValue::Null`
> * Implement the rest of changes in coercion as needed.

i wonder how this line up with typed nulls for other enum cases.

or, could we actually get rid of the `Option` wrapping for other types, and keep this null enum case with wrapper type instead?

```diff
diff --git a/datafusion/src/scalar.rs b/datafusion/src/scalar.rs
index 00586bf55..0ce87195c 100644
--- a/datafusion/src/scalar.rs
+++ b/datafusion/src/scalar.rs
@@ -37,32 +37,34 @@ use std::{convert::TryFrom, fmt, iter::repeat, sync::Arc};
 /// This is the single-valued counter-part of arrow’s `Array`.
 #[derive(Clone)]
 pub enum ScalarValue {
+    /// a typed or untyped null
+    Null(DataType),
     /// true or false value
-    Boolean(Option<bool>),
+    Boolean(bool)
     /// 32bit float
-    Float32(Option<f32>),
+    Float32(f32)
     /// 64bit float
-    Float64(Option<f64>),
+    Float64(f64)
     /// signed 8bit int
-    Int8(Option<i8>),
+    Int8(i8)
     /// signed 16bit int
-    Int16(Option<i16>),
+    Int16(i16)
     /// signed 32bit int
-    Int32(Option<i32>),
+    Int32(i32)
     /// signed 64bit int
-    Int64(Option<i64>),
+    Int64(i64)
     /// unsigned 8bit int
-    UInt8(Option<u8>),
+    UInt8(u8)
     /// unsigned 16bit int
-    UInt16(Option<u16>),
+    UInt16(u16)
     /// unsigned 32bit int
-    UInt32(Option<u32>),
+    UInt32(u32)
     /// unsigned 64bit int
-    UInt64(Option<u64>),
+    UInt64(u64)
     /// utf-8 encoded string.
-    Utf8(Option<String>),
+    Utf8(String)
     /// utf-8 encoded string representing a LargeString's arrow type.
-    LargeUtf8(Option<String>),
+    LargeUtf8(String)
     /// binary
     Binary(Option<Vec<u8>>),
     /// large binary
@@ -71,21 +73,21 @@ pub enum ScalarValue {
     #[allow(clippy::box_vec)]
     List(Option<Box<Vec<ScalarValue>>>, Box<DataType>),
     /// Date stored as a signed 32bit int
-    Date32(Option<i32>),
+    Date32(i32)
     /// Date stored as a signed 64bit int
-    Date64(Option<i64>),
+    Date64(i64)
     /// Timestamp Second
-    TimestampSecond(Option<i64>),
+    TimestampSecond(i64)
     /// Timestamp Milliseconds
-    TimestampMillisecond(Option<i64>),
+    TimestampMillisecond(i64)
     /// Timestamp Microseconds
-    TimestampMicrosecond(Option<i64>),
+    TimestampMicrosecond(i64)
     /// Timestamp Nanoseconds
-    TimestampNanosecond(Option<i64>),
+    TimestampNanosecond(i64)
     /// Interval with YearMonth unit
-    IntervalYearMonth(Option<i32>),
+    IntervalYearMonth(i32)
     /// Interval with DayTime unit
-    IntervalDayTime(Option<i64>),
+    IntervalDayTime(i64)
     /// struct of nested ScalarValue (boxed to reduce size_of(ScalarValue))
     #[allow(clippy::box_vec)]
     Struct(Option<Box<Vec<ScalarValue>>>, Box<Vec<Field>>),
```
> Hey @alamb did you start working on this?
> I might otherwise check it out how this is handled - maybe before / in the weekend.

Thanks @Dandandan  and @Jimexist  -- yes I have started working on this, and came up with something simple (#1181) and uncovered a bunch of other work 

> To introduce ScalarValue::Null and use that instead for Value::Null.
> Return DataType::Null as type for ScalarValue::Null
> Implement the rest of changes in coercion as needed.

Yes I agree that would be the ideal solution -- I tried a variant of that (basically made `ScalarValue::get_datatype()` return DataType::Null if `ScalarValue::is_null()` and taught the coercion logic about it for binary operators).  However, I have since abandoned that approach in the near term.

The biggest problem I hit was that the arrow-rs cast kernels doesn't support casting from `Null` -> other_type -- I will be filing a ticket shortly for that.

I also found that functions and binary expressions don't share the same coercion logic and that many other expressions / operators don't actually try to coerce their arguments at all (e.g. try running a query like `select case when NULL then 'foo' else 'bar' end;`)

I'll also be filing tickets for those items as well
🤔  well I thought #1181  was ready but apparently it is not -- I am about out of time to work on this issue today, so I am going to spend the rest of my time writing up tickets.



Patch:
diff --git a/datafusion/expr/src/binary_rule.rs b/datafusion/expr/src/binary_rule.rs
index 63a9712fd5ac..f3d7534674eb 100644
--- a/datafusion/expr/src/binary_rule.rs
+++ b/datafusion/expr/src/binary_rule.rs
@@ -161,6 +161,7 @@ fn comparison_eq_coercion(lhs_type: &DataType, rhs_type: &DataType) -> Option<Da
         .or_else(|| dictionary_coercion(lhs_type, rhs_type))
         .or_else(|| temporal_coercion(lhs_type, rhs_type))
         .or_else(|| string_coercion(lhs_type, rhs_type))
+        .or_else(|| null_coercion(lhs_type, rhs_type))
 }
 
 fn comparison_order_coercion(
@@ -177,6 +178,7 @@ fn comparison_order_coercion(
         .or_else(|| string_coercion(lhs_type, rhs_type))
         .or_else(|| dictionary_coercion(lhs_type, rhs_type))
         .or_else(|| temporal_coercion(lhs_type, rhs_type))
+        .or_else(|| null_coercion(lhs_type, rhs_type))
 }
 
 fn comparison_binary_numeric_coercion(
diff --git a/datafusion/physical-expr/src/expressions/binary.rs b/datafusion/physical-expr/src/expressions/binary.rs
index 060f30cb2d77..25beea5b22b1 100644
--- a/datafusion/physical-expr/src/expressions/binary.rs
+++ b/datafusion/physical-expr/src/expressions/binary.rs
@@ -657,17 +657,15 @@ macro_rules! compute_utf8_op_scalar {
 
 /// Invoke a compute kernel on a data array and a scalar value
 macro_rules! compute_utf8_op_dyn_scalar {
-    ($LEFT:expr, $RIGHT:expr, $OP:ident) => {{
+    ($LEFT:expr, $RIGHT:expr, $OP:ident, $OP_TYPE:expr) => {{
         if let Some(string_value) = $RIGHT {
             Ok(Arc::new(paste::expr! {[<$OP _dyn_utf8_scalar>]}(
                 $LEFT,
                 &string_value,
             )?))
         } else {
-            Err(DataFusionError::Internal(format!(
-                "compute_utf8_op_scalar for '{}' failed with literal 'none' value",
-                stringify!($OP),
-            )))
+            // when the $RIGHT is a NULL, generate a NULL array of $OP_TYPE
+            Ok(Arc::new(new_null_array($OP_TYPE, $LEFT.len())))
         }
     }};
 }
@@ -691,7 +689,7 @@ macro_rules! compute_bool_op_scalar {
 
 /// Invoke a compute kernel on a boolean data array and a scalar value
 macro_rules! compute_bool_op_dyn_scalar {
-    ($LEFT:expr, $RIGHT:expr, $OP:ident) => {{
+    ($LEFT:expr, $RIGHT:expr, $OP:ident, $OP_TYPE:expr) => {{
         // generate the scalar function name, such as lt_dyn_bool_scalar, from the $OP parameter
         // (which could have a value of lt) and the suffix _scalar
         if let Some(b) = $RIGHT {
@@ -700,10 +698,8 @@ macro_rules! compute_bool_op_dyn_scalar {
                 b,
             )?))
         } else {
-            Err(DataFusionError::Internal(format!(
-                "compute_utf8_op_scalar for '{}' failed with literal 'none' value",
-                stringify!($OP),
-            )))
+            // when the $RIGHT is a NULL, generate a NULL array of $OP_TYPE
+            Ok(Arc::new(new_null_array($OP_TYPE, $LEFT.len())))
         }
     }};
 }
@@ -751,8 +747,9 @@ macro_rules! compute_op_scalar {
 
 /// Invoke a dyn compute kernel on a data array and a scalar value
 /// LEFT is Primitive or Dictionart array of numeric values, RIGHT is scalar value
+/// OP_TYPE is the return type of scalar function
 macro_rules! compute_op_dyn_scalar {
-    ($LEFT:expr, $RIGHT:expr, $OP:ident) => {{
+    ($LEFT:expr, $RIGHT:expr, $OP:ident, $OP_TYPE:expr) => {{
         // generate the scalar function name, such as lt_dyn_scalar, from the $OP parameter
         // (which could have a value of lt_dyn) and the suffix _scalar
         if let Some(value) = $RIGHT {
@@ -761,10 +758,8 @@ macro_rules! compute_op_dyn_scalar {
                 value,
             )?))
         } else {
-            Err(DataFusionError::Internal(format!(
-                "compute_utf8_op_scalar for '{}' failed with literal 'none' value",
-                stringify!($OP),
-            )))
+            // when the $RIGHT is a NULL, generate a NULL array of $OP_TYPE
+            Ok(Arc::new(new_null_array($OP_TYPE, $LEFT.len())))
         }
     }};
 }
@@ -1125,22 +1120,22 @@ impl PhysicalExpr for BinaryExpr {
 /// such as Utf8 strings.
 #[macro_export]
 macro_rules! binary_array_op_dyn_scalar {
-    ($LEFT:expr, $RIGHT:expr, $OP:ident) => {{
+    ($LEFT:expr, $RIGHT:expr, $OP:ident, $OP_TYPE:expr) => {{
         let result: Result<Arc<dyn Array>> = match $RIGHT {
-            ScalarValue::Boolean(b) => compute_bool_op_dyn_scalar!($LEFT, b, $OP),
+            ScalarValue::Boolean(b) => compute_bool_op_dyn_scalar!($LEFT, b, $OP, $OP_TYPE),
             ScalarValue::Decimal128(..) => compute_decimal_op_scalar!($LEFT, $RIGHT, $OP, DecimalArray),
-            ScalarValue::Utf8(v) => compute_utf8_op_dyn_scalar!($LEFT, v, $OP),
-            ScalarValue::LargeUtf8(v) => compute_utf8_op_dyn_scalar!($LEFT, v, $OP),
-            ScalarValue::Int8(v) => compute_op_dyn_scalar!($LEFT, v, $OP),
-            ScalarValue::Int16(v) => compute_op_dyn_scalar!($LEFT, v, $OP),
-            ScalarValue::Int32(v) => compute_op_dyn_scalar!($LEFT, v, $OP),
-            ScalarValue::Int64(v) => compute_op_dyn_scalar!($LEFT, v, $OP),
-            ScalarValue::UInt8(v) => compute_op_dyn_scalar!($LEFT, v, $OP),
-            ScalarValue::UInt16(v) => compute_op_dyn_scalar!($LEFT, v, $OP),
-            ScalarValue::UInt32(v) => compute_op_dyn_scalar!($LEFT, v, $OP),
-            ScalarValue::UInt64(v) => compute_op_dyn_scalar!($LEFT, v, $OP),
-            ScalarValue::Float32(_) => compute_op_scalar!($LEFT, $RIGHT, $OP, Float32Array),
-            ScalarValue::Float64(_) => compute_op_scalar!($LEFT, $RIGHT, $OP, Float64Array),
+            ScalarValue::Utf8(v) => compute_utf8_op_dyn_scalar!($LEFT, v, $OP, $OP_TYPE),
+            ScalarValue::LargeUtf8(v) => compute_utf8_op_dyn_scalar!($LEFT, v, $OP, $OP_TYPE),
+            ScalarValue::Int8(v) => compute_op_dyn_scalar!($LEFT, v, $OP, $OP_TYPE),
+            ScalarValue::Int16(v) => compute_op_dyn_scalar!($LEFT, v, $OP, $OP_TYPE),
+            ScalarValue::Int32(v) => compute_op_dyn_scalar!($LEFT, v, $OP, $OP_TYPE),
+            ScalarValue::Int64(v) => compute_op_dyn_scalar!($LEFT, v, $OP, $OP_TYPE),
+            ScalarValue::UInt8(v) => compute_op_dyn_scalar!($LEFT, v, $OP, $OP_TYPE),
+            ScalarValue::UInt16(v) => compute_op_dyn_scalar!($LEFT, v, $OP, $OP_TYPE),
+            ScalarValue::UInt32(v) => compute_op_dyn_scalar!($LEFT, v, $OP, $OP_TYPE),
+            ScalarValue::UInt64(v) => compute_op_dyn_scalar!($LEFT, v, $OP, $OP_TYPE),
+            ScalarValue::Float32(v) => compute_op_dyn_scalar!($LEFT, v, $OP, $OP_TYPE),
+            ScalarValue::Float64(v) => compute_op_dyn_scalar!($LEFT, v, $OP, $OP_TYPE),
             ScalarValue::Date32(_) => compute_op_scalar!($LEFT, $RIGHT, $OP, Date32Array),
             ScalarValue::Date64(_) => compute_op_scalar!($LEFT, $RIGHT, $OP, Date64Array),
             ScalarValue::TimestampSecond(..) => compute_op_scalar!($LEFT, $RIGHT, $OP, TimestampSecondArray),
@@ -1163,22 +1158,37 @@ impl BinaryExpr {
     ) -> Result<Option<Result<ArrayRef>>> {
         let scalar_result = match &self.op {
             Operator::Lt => {
-                binary_array_op_dyn_scalar!(array, scalar.clone(), lt)
+                binary_array_op_dyn_scalar!(array, scalar.clone(), lt, &DataType::Boolean)
             }
             Operator::LtEq => {
-                binary_array_op_dyn_scalar!(array, scalar.clone(), lt_eq)
+                binary_array_op_dyn_scalar!(
+                    array,
+                    scalar.clone(),
+                    lt_eq,
+                    &DataType::Boolean
+                )
             }
             Operator::Gt => {
-                binary_array_op_dyn_scalar!(array, scalar.clone(), gt)
+                binary_array_op_dyn_scalar!(array, scalar.clone(), gt, &DataType::Boolean)
             }
             Operator::GtEq => {
-                binary_array_op_dyn_scalar!(array, scalar.clone(), gt_eq)
+                binary_array_op_dyn_scalar!(
+                    array,
+                    scalar.clone(),
+                    gt_eq,
+                    &DataType::Boolean
+                )
             }
             Operator::Eq => {
-                binary_array_op_dyn_scalar!(array, scalar.clone(), eq)
+                binary_array_op_dyn_scalar!(array, scalar.clone(), eq, &DataType::Boolean)
             }
             Operator::NotEq => {
-                binary_array_op_dyn_scalar!(array, scalar.clone(), neq)
+                binary_array_op_dyn_scalar!(
+                    array,
+                    scalar.clone(),
+                    neq,
+                    &DataType::Boolean
+                )
             }
             Operator::Like => {
                 binary_string_array_op_scalar!(array, scalar.clone(), like)


Test patch:
diff --git a/datafusion/core/tests/sql/expr.rs b/datafusion/core/tests/sql/expr.rs
index 1dffc2eb9366..3a88dd141d5e 100644
--- a/datafusion/core/tests/sql/expr.rs
+++ b/datafusion/core/tests/sql/expr.rs
@@ -1158,3 +1158,123 @@ async fn nested_subquery() -> Result<()> {
     assert_batches_eq!(expected, &actual);
     Ok(())
 }
+
+#[tokio::test]
+async fn comparisons_with_null() -> Result<()> {
+    let ctx = SessionContext::new();
+    // 1. Numeric comparison with NULL
+    let sql = "select column1 < NULL from (VALUES (1, 'foo' ,2.3), (2, 'bar', 5.4)) as t";
+    let actual = execute_to_batches(&ctx, sql).await;
+    let expected = vec![
+        "+-------------------+",
+        "| t.column1 Lt NULL |",
+        "+-------------------+",
+        "|                   |",
+        "|                   |",
+        "+-------------------+",
+    ];
+    assert_batches_eq!(expected, &actual);
+
+    let sql =
+        "select column1 <= NULL from (VALUES (1, 'foo' ,2.3), (2, 'bar', 5.4)) as t";
+    let actual = execute_to_batches(&ctx, sql).await;
+    let expected = vec![
+        "+---------------------+",
+        "| t.column1 LtEq NULL |",
+        "+---------------------+",
+        "|                     |",
+        "|                     |",
+        "+---------------------+",
+    ];
+    assert_batches_eq!(expected, &actual);
+
+    let sql = "select column1 > NULL from (VALUES (1, 'foo' ,2.3), (2, 'bar', 5.4)) as t";
+    let actual = execute_to_batches(&ctx, sql).await;
+    let expected = vec![
+        "+-------------------+",
+        "| t.column1 Gt NULL |",
+        "+-------------------+",
+        "|                   |",
+        "|                   |",
+        "+-------------------+",
+    ];
+    assert_batches_eq!(expected, &actual);
+
+    let sql =
+        "select column1 >= NULL from (VALUES (1, 'foo' ,2.3), (2, 'bar', 5.4)) as t";
+    let actual = execute_to_batches(&ctx, sql).await;
+    let expected = vec![
+        "+---------------------+",
+        "| t.column1 GtEq NULL |",
+        "+---------------------+",
+        "|                     |",
+        "|                     |",
+        "+---------------------+",
+    ];
+    assert_batches_eq!(expected, &actual);
+
+    let sql = "select column1 = NULL from (VALUES (1, 'foo' ,2.3), (2, 'bar', 5.4)) as t";
+    let actual = execute_to_batches(&ctx, sql).await;
+    let expected = vec![
+        "+-------------------+",
+        "| t.column1 Eq NULL |",
+        "+-------------------+",
+        "|                   |",
+        "|                   |",
+        "+-------------------+",
+    ];
+    assert_batches_eq!(expected, &actual);
+
+    let sql =
+        "select column1 != NULL from (VALUES (1, 'foo' ,2.3), (2, 'bar', 5.4)) as t";
+    let actual = execute_to_batches(&ctx, sql).await;
+    let expected = vec![
+        "+----------------------+",
+        "| t.column1 NotEq NULL |",
+        "+----------------------+",
+        "|                      |",
+        "|                      |",
+        "+----------------------+",
+    ];
+    assert_batches_eq!(expected, &actual);
+
+    // 1.1 Float value comparison with NULL
+    let sql = "select column3 < NULL from (VALUES (1, 'foo' ,2.3), (2, 'bar', 5.4)) as t";
+    let actual = execute_to_batches(&ctx, sql).await;
+    let expected = vec![
+        "+-------------------+",
+        "| t.column3 Lt NULL |",
+        "+-------------------+",
+        "|                   |",
+        "|                   |",
+        "+-------------------+",
+    ];
+    assert_batches_eq!(expected, &actual);
+
+    // String comparison with NULL
+    let sql = "select column2 < NULL from (VALUES (1, 'foo' ,2.3), (2, 'bar', 5.4)) as t";
+    let actual = execute_to_batches(&ctx, sql).await;
+    let expected = vec![
+        "+-------------------+",
+        "| t.column2 Lt NULL |",
+        "+-------------------+",
+        "|                   |",
+        "|                   |",
+        "+-------------------+",
+    ];
+    assert_batches_eq!(expected, &actual);
+
+    // Boolean comparison with NULL
+    let sql = "select column1 < NULL from (VALUES (true), (false)) as t";
+    let actual = execute_to_batches(&ctx, sql).await;
+    let expected = vec![
+        "+-------------------+",
+        "| t.column1 Lt NULL |",
+        "+-------------------+",
+        "|                   |",
+        "|                   |",
+        "+-------------------+",
+    ];
+    assert_batches_eq!(expected, &actual);
+    Ok(())
+}