Repository: ibis-project/ibis

Issue:
bug(duckdb) reading parquet with multi-dim array yields ParseError: expected 'EOF' on `conn.table(`
## Issue
Saving a multi-dimensional array to parquet and then attempting to read the table post-register yields a ParseError.
Not sure why.

Expected behavior: able to connect to a table expression (and then subsequently execute on it)

Actual behavior: `ParseError: expected 'EOF' at 0:8` on `conn.table("fields")`.

I think this is an Ibis issue specifically since I can query using `duckdb.connect().cursor().execute(`, but I'm not 100% sure.
Code below:

## Setup

```python
import os
import glob

import duckdb
## Found using ibis
import ibis
import pandas as pd

FARM_DATA_DIR = 'farm_data'


def write_data(data, fname, dname=FARM_DATA_DIR, fmt='parquet'):
    if not os.path.exists(dname):
        os.mkdir(dname)
    path = '.'.join([os.path.join(dname, fname), fmt])

    try:
        os.remove(path)
    except:
        pass

    if fmt in ['csv', 'txt']:
        with open(path, 'w') as f:
            for row in data:
                line = '|'.join(str(e) for e in row)
                f.write(line)
                f.write('\n')
    elif fmt in ['parquet']:
        df = pd.DataFrame(data[1:], columns=data[0])
        df.to_parquet(path, index=False)
    return path

fields = [
    ['field_id', 'field_tract_id', 'field_vertices']
    ,[1, 1, [[0, 2], [1, 3], [2, 0], [3, 1]]]
    ,[2, 1, [[2, 2], [3, 2], [3, 1]]]
    ,[3, 2, [[0, 1], [-1, 0], [-2, 0], [-2, 1]]]
    ,[4, 3, [[0, 1], [1, 1], [1, 2], [0, 2]]]
    ,[5, 3, [[1, 0], [2, 0], [2, 3], [1, 3]]]
    ,[6, 3, [[2, 0], [3, 0], [3, 2], [2, 2]]]
    ,[7, 4, [[-1, -1], [0, -1], [0, -2]]]
    ,[8, 4, [[1, 0], [1, -2], [0, -2], [0, -1]]]
    ,[9, 5, [[1, 0], [2, 0], [1, -1], [1, -2], [-1, -2], [-1, -1], [-2, 0], [-1, 0], [0, -1]]]
    ,[10, 6, [[-1, 2], [0, 2], [0, 0], [-1, 0]]]
    ,[11, 6, [[0, 2], [1, 2], [1, 1], [0, 1]]]
]

file = write_data(fields, 'fields')
```

Ibis:

```python
conn = ibis.connect("duckdb://:memory:")

conn.register("farm_data/fields.parquet", "fields")
conn.table("fields")
```

Error:
```python
---------------------------------------------------------------------------
ParseError                                Traceback (most recent call last)
Input In [13], in <cell line: 1>()
----> 1 conn.table("fields")

File ~/dev/repos/ibis/ibis/backends/base/sql/alchemy/__init__.py:412, in BaseAlchemyBackend.table(self, name, database, schema)
    406 if database is not None and database != self.current_database:
    407     return self.database(database=database).table(
    408         name=name,
    409         database=database,
    410         schema=schema,
    411     )
--> 412 sqla_table = self._get_sqla_table(
    413     name,
    414     database=database,
    415     schema=schema,
    416 )
    417 return self._sqla_table_to_expr(sqla_table)

File ~/dev/repos/ibis/ibis/backends/duckdb/__init__.py:187, in Backend._get_sqla_table(self, name, schema, **kwargs)
    183     return table
    185 quoted_name = self.con.dialect.identifier_preparer.quote(name)
--> 187 for colname, type in self._metadata(quoted_name):
    188     if colname in nulltype_cols:
    189         # replace null types discovered by sqlalchemy with non null
    190         # types
    191         table.append_column(
    192             sa.Column(
    193                 colname,
   (...)
    197             replace_existing=True,
    198         )

File ~/dev/repos/ibis/ibis/backends/duckdb/__init__.py:153, in Backend._metadata(self, query)
    146 def _metadata(self, query: str) -> Iterator[_ColumnMetadata]:
    147     for name, type, null in toolz.pluck(
    148         ["column_name", "column_type", "null"],
    149         self.con.execute(f"DESCRIBE {query}"),
    150     ):
    151         yield _ColumnMetadata(
    152             name=name,
--> 153             type=parse(type)(nullable=null.lower() == "yes"),
    154         )

File ~/dev/repos/ibis/ibis/backends/duckdb/datatypes.py:146, in parse(text, default_decimal_parameters)
    144 non_pg_array_type = primitive | decimal | list_array | map | struct
    145 ty = pg_array | non_pg_array_type
--> 146 return ty.parse(text)

File ~/dev/envs/dev/lib/python3.8/site-packages/parsy/__init__.py:90, in Parser.parse(self, stream)
     88 def parse(self, stream):
     89     """Parse a string or list of tokens and return the result or raise a ParseError."""
---> 90     (result, _) = (self << eof).parse_partial(stream)
     91     return result

File ~/dev/envs/dev/lib/python3.8/site-packages/parsy/__init__.py:104, in Parser.parse_partial(self, stream)
    102     return (result.value, stream[result.index:])
    103 else:
--> 104     raise ParseError(result.expected, stream, result.furthest)

ParseError: expected 'EOF' at 0:8
```

DuckDB raw:
```python
sql = "describe select * from 'farm_data/fields.parquet'"
cur = conn.cursor()
result = cur.execute(sql).fetchall()
print("describe:\n", result)

sql = "select * from 'farm_data/fields.parquet'"
cur = conn.cursor()
result = cur.execute(sql).fetchall()
print("\nselect:\n", result)

>>> describe:
>>> [('field_id', 'BIGINT', 'YES', None, None, None), ('field_tract_id', 'BIGINT', 'YES', None, None, None), ('field_vertices', 'BIGINT[][]', 'YES', None, None, None)]

>>> select:
>>> [(1, 1, [[0, 2], [1, 3], [2, 0], [3, 1]]), (2, 1, [[2, 2], [3, 2], [3, 1]]), (3, 2, [[0, 1], [-1, 0], [-2, 0], [-2, 1]]), (4, 3, [[0, 1], [1, 1], [1, 2], [0, 2]]), (5, 3, [[1, 0], [2, 0], [2, 3], [1, 3]]), (6, 3, [[2, 0], [3, 0], [3, 2], [2, 2]]), (7, 4, [[-1, -1], [0, -1], [0, -2]]), (8, 4, [[1, 0], [1, -2], [0, -2], [0, -1]]), (9, 5, [[1, 0], [2, 0], [1, -1], [1, -2], [-1, -2], [-1, -1], [-2, 0], [-1, 0], [0, -1]]), (10, 6, [[-1, 2], [0, 2], [0, 0], [-1, 0]]), (11, 6, [[0, 2], [1, 2], [1, 1], [0, 1]])]
```


Discussion:
Yeah, the duckdb type parser probably doesn't handle repeated `[]`s.

Resolving patch:
diff --git a/ibis/backends/base/sql/alchemy/datatypes.py b/ibis/backends/base/sql/alchemy/datatypes.py
--- a/ibis/backends/base/sql/alchemy/datatypes.py
+++ b/ibis/backends/base/sql/alchemy/datatypes.py
@@ -95,8 +95,11 @@ def _(itype, **kwargs):
 
 @to_sqla_type.register(dt.Array)
 def _(itype, **kwargs):
-    ibis_type = itype.value_type
-    return sa.ARRAY(to_sqla_type(ibis_type, **kwargs))
+    # Unwrap the array element type because sqlalchemy doesn't allow arrays of
+    # arrays. This doesn't affect the underlying data.
+    while isinstance(itype, dt.Array):
+        itype = itype.value_type
+    return sa.ARRAY(to_sqla_type(itype, **kwargs))
 
 
 @to_sqla_type.register(dt.Struct)
diff --git a/ibis/backends/duckdb/datatypes.py b/ibis/backends/duckdb/datatypes.py
--- a/ibis/backends/duckdb/datatypes.py
+++ b/ibis/backends/duckdb/datatypes.py
@@ -3,6 +3,7 @@
 from typing import TYPE_CHECKING
 
 import parsy as p
+import toolz
 
 from ibis import util
 
@@ -110,12 +111,16 @@ def list_array():
         value_type = yield angle_type
         return Array(value_type)
 
+    @p.generate
+    def brackets():
+        yield spaceless(LBRACKET)
+        yield spaceless(RBRACKET)
+
     @p.generate
     def pg_array():
         value_type = yield non_pg_array_type
-        yield LBRACKET
-        yield RBRACKET
-        return Array(value_type)
+        n = len((yield brackets.at_least(1)))
+        return toolz.nth(n, toolz.iterate(Array, value_type))
 
     @p.generate
     def map():


Test patch:
diff --git a/ibis/backends/duckdb/tests/test_datatypes.py b/ibis/backends/duckdb/tests/test_datatypes.py
--- a/ibis/backends/duckdb/tests/test_datatypes.py
+++ b/ibis/backends/duckdb/tests/test_datatypes.py
@@ -56,6 +56,8 @@
             c=dt.Array(dt.Map(dt.string, dt.Array(dt.float64))),
         )
     ),
+    T=dt.Array(dt.Array(dt.int32)),
+    U=dt.Array(dt.Array(dt.int32)),
 )
 
 
@@ -109,6 +111,8 @@
             ("Q", "LIST<INTEGER>"),
             ("R", "MAP<STRING, BIGINT>"),
             ("S", "STRUCT(a INT, b TEXT, c LIST<MAP<TEXT, LIST<FLOAT8>>>)"),
+            ("T", "LIST<LIST<INTEGER>>"),
+            ("U", "INTEGER[][]"),
         ]
     ],
 )
diff --git a/ibis/backends/pandas/tests/conftest.py b/ibis/backends/pandas/tests/conftest.py
--- a/ibis/backends/pandas/tests/conftest.py
+++ b/ibis/backends/pandas/tests/conftest.py
@@ -54,6 +54,7 @@ def connect(data_directory: Path):
                             [1.0, 2.0, 3.0],
                             'a',
                             1.0,
+                            [[], [np.int64(1), 2, 3], None],
                         ),
                         (
                             [4, 5],
@@ -61,6 +62,7 @@ def connect(data_directory: Path):
                             [4.0, 5.0],
                             'a',
                             2.0,
+                            [],
                         ),
                         (
                             [6, None],
@@ -68,6 +70,7 @@ def connect(data_directory: Path):
                             [6.0, np.nan],
                             'a',
                             3.0,
+                            [None, [], None],
                         ),
                         (
                             [None, 1, None],
@@ -75,6 +78,7 @@ def connect(data_directory: Path):
                             [],
                             'b',
                             4.0,
+                            [[1], [2], [], [3, 4, 5]],
                         ),
                         (
                             [2, None, 3],
@@ -82,6 +86,7 @@ def connect(data_directory: Path):
                             np.nan,
                             'b',
                             5.0,
+                            None,
                         ),
                         (
                             [4, None, None, 5],
@@ -89,9 +94,17 @@ def connect(data_directory: Path):
                             [4.0, np.nan, np.nan, 5.0],
                             'c',
                             6.0,
+                            [[1, 2, 3]],
                         ),
                     ],
-                    columns=["x", "y", "z", "grouper", "scalar_column"],
+                    columns=[
+                        "x",
+                        "y",
+                        "z",
+                        "grouper",
+                        "scalar_column",
+                        "multi_dim",
+                    ],
                 ),
             }
         )
diff --git a/ibis/backends/pyspark/tests/conftest.py b/ibis/backends/pyspark/tests/conftest.py
--- a/ibis/backends/pyspark/tests/conftest.py
+++ b/ibis/backends/pyspark/tests/conftest.py
@@ -152,20 +152,35 @@ def get_common_spark_testing_client(data_directory, connect):
     df_nested_types.createOrReplaceTempView('nested_types')
     df_array_types = s.createDataFrame(
         [
-            ([1, 2, 3], ['a', 'b', 'c'], [1.0, 2.0, 3.0], 'a', 1.0),
-            ([4, 5], ['d', 'e'], [4.0, 5.0], 'a', 2.0),
-            ([6, None], ['f', None], [6.0, None], 'a', 3.0),
-            ([None, 1, None], [None, 'a', None], [], 'b', 4.0),
-            ([2, None, 3], ['b', None, 'c'], None, 'b', 5.0),
+            (
+                [1, 2, 3],
+                ['a', 'b', 'c'],
+                [1.0, 2.0, 3.0],
+                'a',
+                1.0,
+                [[], [1, 2, 3], None],
+            ),
+            ([4, 5], ['d', 'e'], [4.0, 5.0], 'a', 2.0, []),
+            ([6, None], ['f', None], [6.0, None], 'a', 3.0, [None, [], None]),
+            (
+                [None, 1, None],
+                [None, 'a', None],
+                [],
+                'b',
+                4.0,
+                [[1], [2], [], [3, 4, 5]],
+            ),
+            ([2, None, 3], ['b', None, 'c'], None, 'b', 5.0, None),
             (
                 [4, None, None, 5],
                 ['d', None, None, 'e'],
                 [4.0, None, None, 5.0],
                 'c',
                 6.0,
+                [[1, 2, 3]],
             ),
         ],
-        ["x", "y", "z", "grouper", "scalar_column"],
+        ["x", "y", "z", "grouper", "scalar_column", "multi_dim"],
     )
     df_array_types.createOrReplaceTempView("array_types")
 
diff --git a/ibis/backends/tests/test_array.py b/ibis/backends/tests/test_array.py
--- a/ibis/backends/tests/test_array.py
+++ b/ibis/backends/tests/test_array.py
@@ -125,7 +125,54 @@ def test_np_array_literal(con):
 
 
 @builtin_array
-def test_array_discovery(con):
+@pytest.mark.never(
+    ["clickhouse", "pandas", "pyspark"],
+    reason="backend does not flatten array types",
+)
+def test_array_discovery_postgres_duckdb(con):
+    t = con.table("array_types")
+    expected = ibis.schema(
+        dict(
+            x=dt.Array(dt.int64),
+            y=dt.Array(dt.string),
+            z=dt.Array(dt.float64),
+            grouper=dt.string,
+            scalar_column=dt.float64,
+            multi_dim=dt.Array(dt.int64),
+        )
+    )
+    assert t.schema() == expected
+
+
+@builtin_array
+@pytest.mark.never(
+    ["duckdb", "pandas", "postgres", "pyspark"],
+    reason="backend supports nullable nested types",
+)
+def test_array_discovery_clickhouse(con):
+    t = con.table("array_types")
+    expected = ibis.schema(
+        dict(
+            x=dt.Array(dt.int64, nullable=False),
+            y=dt.Array(dt.string, nullable=False),
+            z=dt.Array(dt.float64, nullable=False),
+            grouper=dt.string,
+            scalar_column=dt.float64,
+            multi_dim=dt.Array(
+                dt.Array(dt.int64, nullable=False),
+                nullable=False,
+            ),
+        )
+    )
+    assert t.schema() == expected
+
+
+@builtin_array
+@pytest.mark.notyet(
+    ["clickhouse", "duckdb", "postgres"],
+    reason="backend does not support nullable nested types",
+)
+def test_array_discovery_desired(con):
     t = con.table("array_types")
     expected = ibis.schema(
         dict(
@@ -134,11 +181,10 @@ def test_array_discovery(con):
             z=dt.Array(dt.float64),
             grouper=dt.string,
             scalar_column=dt.float64,
+            multi_dim=dt.Array(dt.Array(dt.int64)),
         )
     )
-    assert list(t.columns) == list(expected.names)
-    for name, typ in t.schema().items():
-        assert isinstance(typ, type(expected[name]))
+    assert t.schema() == expected
 
 
 @unnest