Repository: ibis-project/ibis

Issue:
bug: cannot compute precedence for `non-nullable` column and `null` in `ifelse` expression.
Hi, I found a bug of type casting validation in `ifelse` expression.

Let's see an example:

```python
"""
This exapmle is shown on clickhouse backends and `ibis-testing-data`, but also effective on other backends, such as mysql.
"""
import ibis

connection = ibis.clickhouse.connect(
    host='127.0.0.1',
    database='ibis_testing',
    port=9000
)

table_alltypes = connection.table('functional_alltypes')
boolean_col = table_alltypes.string_col == '1'

# The following expression will raise:
#   IbisTypeError: Cannot compute precedence for string[non-nullable] and null types
expr = ibis.ifelse(boolean_col, table_alltypes.string_col, ibis.NA)


# The expected sql compiled from `expr`
sql = """
SELECT CASE WHEN `string_col` = '1' THEN `string_col` ELSE null END AS `tmp`
FROM ibis_testing.`functional_alltypes`
"""

# Can get the right result
connection.raw_sql(sql)
```

If `string_col` is `nullable`, ibis will work well.

It's a bug? Or I used ibis in the wrong way🤒?



Discussion:
This is definitely an issue. Non-nullable types are kind of second-class citizens at the moment.

Resolving patch:
diff --git a/ibis/expr/datatypes/core.py b/ibis/expr/datatypes/core.py
--- a/ibis/expr/datatypes/core.py
+++ b/ibis/expr/datatypes/core.py
@@ -1234,13 +1234,15 @@ def _get_timedelta_units(
 
 
 def higher_precedence(left: DataType, right: DataType) -> DataType:
+    nullable = left.nullable or right.nullable
+
     if castable(left, right, upcast=True):
-        return right
+        return right(nullable=nullable)
     elif castable(right, left, upcast=True):
-        return left
+        return left(nullable=nullable)
 
     raise IbisTypeError(
-        f'Cannot compute precedence for {left} and {right} types'
+        f'Cannot compute precedence for `{left}` and `{right}` types'
     )
 
 
@@ -1421,7 +1423,17 @@ def can_cast_any(source: DataType, target: DataType, **kwargs) -> bool:
 
 @castable.register(Null, DataType)
 def can_cast_null(source: DataType, target: DataType, **kwargs) -> bool:
-    return target.nullable
+    # The null type is castable to any type, even if the target type is *not*
+    # nullable.
+    #
+    # We handle the promotion of `null + !T -> T` at the `castable` call site.
+    #
+    # It might be possible to build a system with a single function that tries
+    # to promote types and use the exception to indicate castability, but that
+    # is a deeper refactor to be tackled later.
+    #
+    # See https://github.com/ibis-project/ibis/issues/2891 for the bug report
+    return True
 
 
 Integral = TypeVar('Integral', SignedInteger, UnsignedInteger)
diff --git a/ibis/expr/operations/logical.py b/ibis/expr/operations/logical.py
--- a/ibis/expr/operations/logical.py
+++ b/ibis/expr/operations/logical.py
@@ -8,6 +8,7 @@
 
 from public import public
 
+from ibis.common.validators import immutable_property
 from ibis.expr import datatypes as dt
 from ibis.expr import rules as rlz
 from ibis.expr.operations.core import Binary, Unary, Value
@@ -164,7 +165,15 @@ class Where(Value):
     true_expr = rlz.any
     false_null_expr = rlz.any
 
-    output_dtype = rlz.dtype_like("true_expr")
+    @immutable_property
+    def output_dtype(self):
+        return rlz.highest_precedence_dtype(
+            [
+                self.true_expr,
+                self.false_null_expr,
+            ]
+        )
+
     output_shape = rlz.shape_like("bool_expr")
 
 


Test patch:
diff --git a/ibis/tests/expr/test_value_exprs.py b/ibis/tests/expr/test_value_exprs.py
--- a/ibis/tests/expr/test_value_exprs.py
+++ b/ibis/tests/expr/test_value_exprs.py
@@ -1505,3 +1505,22 @@ def test_deferred_r_ops(op_name, expected_left, expected_right):
     op = expr.op().selections[0].op().arg.op()
     assert op.left.equals(expected_left(t))
     assert op.right.equals(expected_right(t))
+
+
+@pytest.mark.parametrize(
+    ("expr_fn", "expected_type"),
+    [
+        (lambda t: ibis.where(t.a == 1, t.b, ibis.NA), dt.string),
+        (lambda t: ibis.where(t.a == 1, t.b, t.a.cast("string")), dt.string),
+        (
+            lambda t: ibis.where(t.a == 1, t.b, t.a.cast("!string")),
+            dt.string(nullable=False),
+        ),
+        (lambda _: ibis.where(True, ibis.NA, ibis.NA), dt.null),
+        (lambda _: ibis.where(False, ibis.NA, ibis.NA), dt.null),
+    ],
+)
+def test_non_null_with_null_precedence(expr_fn, expected_type):
+    t = ibis.table(dict(a="int64", b="!string"), name="t")
+    expr = expr_fn(t)
+    assert expr.type() == expected_type