Repository: ibis-project/ibis

Issue:
feat: add workarounds for duckdb MAP bugs
### Is your feature request related to a problem?

see
https://github.com/duckdb/duckdb/issues/11115
and
https://github.com/duckdb/duckdb/issues/11116

This might be out of scope, and we should just wait for duckdb to fix these upstream, but we could add some workarounds for these bugs in ibis. I am running into these and am having to implement my own little wrappers, eg

```python
def _map_values(m: ir.MapValue) -> ir.ArrayValue:
    """workaround for https://github.com/duckdb/duckdb/issues/11116"""
    normal = m.values()
    null = ibis.literal(None, type=dt.Array(value_type=m.type().value_type))
    return m.isnull().ifelse(null, normal)
```

feel free to close this out if you don't want to take this on (I don't blame you if you don't 😉 )

### Describe the solution you'd like

I'm not sure where in the stack these fixes would need to go, at compile time, or just when creating expressions.

### What version of ibis are you running?

main

### What backend(s) are you using, if any?

_No response_

### Code of Conduct

- [X] I agree to follow this project's Code of Conduct


Discussion:
Thanks! Looks like you've gone down a map 🐰 🕳️ 

I think it's reasonable to add these workarounds for now.
yeah seriously talk about a 🐰 🕳️ . I was convinced for hours it was bugs in my code wheeeeee

here is what I currently use. I have unit tests for functions that use these, but I don't have unit tests for these as individuals.

```python
def map_keys(m: ir.MapValue) -> ir.ArrayValue:
    """workaround for https://github.com/duckdb/duckdb/issues/11116"""
    normal = m.keys()
    null = ibis.literal(None, type=dt.Array(value_type=m.type().key_type))
    return m.isnull().ifelse(null, normal)


def map_values(m: ir.MapValue) -> ir.ArrayValue:
    """workaround for https://github.com/duckdb/duckdb/issues/11116"""
    normal = m.values()
    null = ibis.literal(None, type=dt.Array(value_type=m.type().value_type))
    return m.isnull().ifelse(null, normal)


def map_(keys: ir.ArrayValue, values: ir.ArrayValue) -> ir.MapValue:
    """workaround for https://github.com/duckdb/duckdb/issues/11115"""
    either_null = keys.isnull() | values.isnull()
    regular = ibis.map(keys, values)
    null = ibis.literal(
        None,
        type=dt.Map(
            key_type=keys.type().value_type, value_type=values.type().value_type
        ),
    )
    return either_null.ifelse(null, regular)
```
Yeah, this would be good to get in for 9.0.  

@NickCrews do you have the bandwidth to PR this in?  We probably do want unit tests for them, and probably also some xfailed tests that will start to XPASS when the upstream issues are fixed.
In that PR I addressed https://github.com/duckdb/duckdb/issues/11115, which
- is more common
- leads to incorrect results in the NULL NULL case (!)

Once that is in then I can tackle https://github.com/duckdb/duckdb/issues/11116, which I think
- is much more rare
- usually leads to errors instead of incorrect reuslts.
I'll work on getting the rest of the map family of functions behaving consistently here.

Resolving patch:
diff --git a/ibis/backends/duckdb/compiler.py b/ibis/backends/duckdb/compiler.py
--- a/ibis/backends/duckdb/compiler.py
+++ b/ibis/backends/duckdb/compiler.py
@@ -43,10 +43,7 @@ class DuckDBCompiler(SQLGlotCompiler):
         ops.Hash: "hash",
         ops.IntegerRange: "range",
         ops.TimestampRange: "range",
-        ops.MapKeys: "map_keys",
         ops.MapLength: "cardinality",
-        ops.MapMerge: "map_concat",
-        ops.MapValues: "map_values",
         ops.Mode: "mode",
         ops.TimeFromHMS: "make_time",
         ops.TypeOf: "typeof",
@@ -201,17 +198,43 @@ def visit_ArrayZip(self, op, *, arg):
 
     def visit_Map(self, op, *, keys, values):
         # workaround for https://github.com/ibis-project/ibis/issues/8632
-        regular = self.f.map(keys, values)
-        either_null = sg.or_(keys.is_(NULL), values.is_(NULL))
-        return self.if_(either_null, NULL, regular)
+        return self.if_(
+            sg.or_(keys.is_(NULL), values.is_(NULL)), NULL, self.f.map(keys, values)
+        )
 
     def visit_MapGet(self, op, *, arg, key, default):
-        return self.f.ifnull(
-            self.f.list_extract(self.f.element_at(arg, key), 1), default
+        return self.if_(
+            arg.is_(NULL),
+            NULL,
+            self.f.ifnull(
+                self.f.list_extract(
+                    self.if_(key.is_(NULL), NULL, self.f.element_at(arg, key)), 1
+                ),
+                default,
+            ),
         )
 
     def visit_MapContains(self, op, *, arg, key):
-        return self.f.len(self.f.element_at(arg, key)).neq(0)
+        return self.if_(
+            arg.is_(NULL),
+            NULL,
+            self.f.len(self.if_(key.is_(NULL), NULL, self.f.element_at(arg, key))).neq(
+                0
+            ),
+        )
+
+    def visit_MapKeys(self, op, *, arg):
+        return self.if_(arg.is_(NULL), NULL, self.f.map_keys(arg))
+
+    def visit_MapValues(self, op, *, arg):
+        return self.if_(arg.is_(NULL), NULL, self.f.map_values(arg))
+
+    def visit_MapMerge(self, op, *, left, right):
+        return self.if_(
+            sg.or_(left.is_(NULL), right.is_(NULL)),
+            NULL,
+            self.f.map_concat(left, right),
+        )
 
     def visit_ToJSONMap(self, op, *, arg):
         return self.if_(
diff --git a/ibis/expr/types/maps.py b/ibis/expr/types/maps.py
--- a/ibis/expr/types/maps.py
+++ b/ibis/expr/types/maps.py
@@ -76,7 +76,7 @@ class MapValue(Value):
     ├───────────────────┤
     │                 2 │
     │                 0 │
-    │                 0 │
+    │              NULL │
     └───────────────────┘
     """
 
@@ -149,7 +149,7 @@ def get(self, key: ir.Value, default: ir.Value | None = None) -> ir.Value:
         ├───────────────────┤
         │                 2 │
         │                 0 │
-        │                 0 │
+        │              NULL │
         └───────────────────┘
         """
 
@@ -304,7 +304,7 @@ def contains(
         ├─────────────────────┤
         │ True                │
         │ False               │
-        │ False               │
+        │ NULL                │
         └─────────────────────┘
         """
         return ops.MapContains(self, key).to_expr()


Test patch:
diff --git a/ibis/backends/tests/test_map.py b/ibis/backends/tests/test_map.py
--- a/ibis/backends/tests/test_map.py
+++ b/ibis/backends/tests/test_map.py
@@ -61,6 +61,177 @@ def test_map_nulls(con, k, v):
     assert con.execute(m) is None
 
 
+@pytest.mark.notyet("clickhouse", reason="nested types can't be NULL")
+@pytest.mark.broken(["pandas", "dask"], reason="TypeError: iteration over a 0-d array")
+@pytest.mark.notimpl(
+    ["risingwave"],
+    raises=PsycoPg2InternalError,
+    reason="function hstore(character varying[], character varying[]) does not exist",
+)
+@pytest.mark.parametrize(
+    ("k", "v"),
+    [
+        param(None, ["c", "d"], id="null_keys"),
+        param(None, None, id="null_both"),
+    ],
+)
+def test_map_keys_nulls(con, k, v):
+    k = ibis.literal(k, type="array<string>")
+    v = ibis.literal(v, type="array<string>")
+    m = ibis.map(k, v)
+    assert con.execute(m.keys()) is None
+
+
+@pytest.mark.notyet("clickhouse", reason="nested types can't be NULL")
+@pytest.mark.notimpl(
+    ["risingwave"],
+    raises=PsycoPg2InternalError,
+    reason="function hstore(character varying[], character varying[]) does not exist",
+)
+@pytest.mark.parametrize(
+    "map",
+    [
+        param(
+            ibis.map(
+                ibis.literal(["a", "b"]), ibis.literal(None, type="array<string>")
+            ),
+            marks=[
+                pytest.mark.broken(
+                    ["pandas", "dask"], reason="TypeError: iteration over a 0-d array"
+                )
+            ],
+            id="null_values",
+        ),
+        param(
+            ibis.map(
+                ibis.literal(None, type="array<string>"),
+                ibis.literal(None, type="array<string>"),
+            ),
+            marks=[
+                pytest.mark.broken(
+                    ["pandas", "dask"], reason="TypeError: iteration over a 0-d array"
+                )
+            ],
+            id="null_both",
+        ),
+        param(ibis.literal(None, type="map<string, string>"), id="null_map"),
+    ],
+)
+def test_map_values_nulls(con, map):
+    assert con.execute(map.values()) is None
+
+
+@pytest.mark.notimpl(
+    ["risingwave"],
+    raises=PsycoPg2InternalError,
+    reason="function hstore(character varying[], character varying[]) does not exist",
+)
+@pytest.mark.parametrize(
+    ("map", "key"),
+    [
+        param(
+            ibis.map(
+                ibis.literal(["a", "b"]), ibis.literal(["c", "d"], type="array<string>")
+            ),
+            ibis.literal(None, type="string"),
+            marks=[
+                pytest.mark.broken(
+                    ["pandas", "dask"],
+                    reason="result is False instead of None",
+                    strict=False,  # passes for contains, but not for get
+                ),
+                pytest.mark.notimpl(
+                    "flink",
+                    raises=AssertionError,
+                    reason="not yet implemented",
+                    strict=False,
+                ),
+            ],
+            id="non_null_map_null_key",
+        ),
+        param(
+            ibis.map(
+                ibis.literal(None, type="array<string>"),
+                ibis.literal(None, type="array<string>"),
+            ),
+            "a",
+            marks=[
+                pytest.mark.notyet("clickhouse", reason="nested types can't be NULL"),
+                pytest.mark.broken(
+                    ["pandas", "dask"], reason="TypeError: iteration over a 0-d array"
+                ),
+            ],
+            id="null_both_non_null_key",
+        ),
+        param(
+            ibis.map(
+                ibis.literal(None, type="array<string>"),
+                ibis.literal(None, type="array<string>"),
+            ),
+            ibis.literal(None, type="string"),
+            marks=[
+                pytest.mark.notyet("clickhouse", reason="nested types can't be NULL"),
+                pytest.mark.broken(
+                    ["pandas", "dask"], reason="TypeError: iteration over a 0-d array"
+                ),
+            ],
+            id="null_both_null_key",
+        ),
+        param(
+            ibis.literal(None, type="map<string, string>"),
+            "a",
+            marks=[
+                pytest.mark.notyet("clickhouse", reason="nested types can't be NULL")
+            ],
+            id="null_map_non_null_key",
+        ),
+        param(
+            ibis.literal(None, type="map<string, string>"),
+            ibis.literal(None, type="string"),
+            marks=[
+                pytest.mark.notyet("clickhouse", reason="nested types can't be NULL")
+            ],
+            id="null_map_null_key",
+        ),
+    ],
+)
+@pytest.mark.parametrize("method", ["get", "contains"])
+def test_map_get_contains_nulls(con, map, key, method):
+    expr = getattr(map, method)
+    assert con.execute(expr(key)) is None
+
+
+@pytest.mark.notyet("clickhouse", reason="nested types can't be NULL")
+@pytest.mark.notimpl(
+    ["risingwave"],
+    raises=PsycoPg2InternalError,
+    reason="function hstore(character varying[], character varying[]) does not exist",
+)
+@pytest.mark.parametrize(
+    ("m1", "m2"),
+    [
+        param(
+            ibis.literal(None, type="map<string, string>"),
+            ibis.literal({"a": "b"}, type="map<string, string>"),
+            id="null_and_non_null",
+        ),
+        param(
+            ibis.literal({"a": "b"}, type="map<string, string>"),
+            ibis.literal(None, type="map<string, string>"),
+            id="non_null_and_null",
+        ),
+        param(
+            ibis.literal(None, type="map<string, string>"),
+            ibis.literal(None, type="map<string, string>"),
+            id="null_and_null",
+        ),
+    ],
+)
+def test_map_merge_nulls(con, m1, m2):
+    concatted = m1 + m2
+    assert con.execute(concatted) is None
+
+
 @pytest.mark.notimpl(["pandas", "dask"])
 def test_map_table(backend):
     table = backend.map