Repository: astropy/astropy

Issue:
# `to_pandas` should support multidimensional columns

### What is the problem this feature will solve?

When a column contains arrays we get this error basically saying that pandas does not support multidimensional columns:
```
In [2]: t = Table({"a": ["foo", "bar"], "b":[[1,2],[3,4]]})
   ...: t.to_pandas()
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
Cell In[2], line 2
      1 t = Table({"a": ["foo", "bar"], "b":[[1,2],[3,4]]})
----> 2 t.to_pandas()

File ~/.pyenv/versions/3.13.1/envs/astropy71/lib/python3.13/site-packages/astropy/table/table.py:4155, in Table.to_pandas(self, index, use_nullable_int)
   4153 badcols = [name for name, col in self.columns.items() if len(col.shape) > 1]
   4154 if badcols:
-> 4155     raise ValueError(
   4156         f"Cannot convert a table with multidimensional columns to a "
   4157         f"pandas DataFrame. Offending columns are: {badcols}\n"
   4158         f"One can filter out such columns using:\n"
   4159         f"names = [name for name in tbl.colnames if len(tbl[name].shape) <= 1]\n"
   4160         f"tbl[names].to_pandas(...)"
   4161     )
   4163 out = OrderedDict()
   4165 for name, column in tbl.columns.items():

ValueError: Cannot convert a table with multidimensional columns to a pandas DataFrame. Offending columns are: ['b']
One can filter out such columns using:
names = [name for name in tbl.colnames if len(tbl[name].shape) <= 1]
tbl[names].to_pandas(...)
```

However it is possible to use "object" columns for this in pandas, which is maybe not as powerful as a real array but still quite practical in some cases. And the funny thing is that astropy will happily convert to pandas if the column contains arrays of different shapes:
```
In [3]: t = Table({"a": ["foo", "bar"], "b":[[1,2],[3,4,5]]})
   ...: t.to_pandas()
Out[3]: 
     a          b
0  foo     [1, 2]
1  bar  [3, 4, 5]
```
So it's a bit sad that it's not possible to get the same thing when arrays have the same shape (I tried if I could trick it but didn't find a way).

### Describe the desired outcome

`Table({"a": ["foo", "bar"], "b":[[1,2],[3,4]]}).to_pandas()` should work.

### Additional context

_No response_

Implementation patch:
diff --git a/astropy/table/_dataframes.py b/astropy/table/_dataframes.py
index 15eeb1b784cc..b09a6fe03c8e 100644
--- a/astropy/table/_dataframes.py
+++ b/astropy/table/_dataframes.py
@@ -169,6 +169,24 @@ def _handle_index_argument(
         )
 
 
+def _convert_multidim_columns_to_object(table: Table) -> Table:
+    """
+    Convert any multidimensional columns in the table to 1D object arrays.
+
+    If there are no multidimensional columns, the table is returned unchanged.
+    """
+    if not any(col.ndim > 1 for col in table.itercols()):
+        return table
+
+    out = table.copy(copy_data=False)
+    for name, col in table.columns.items():
+        if col.ndim > 1:
+            # Convert to 1D object array (e.g. list of lists for ndim=2).
+            out[name] = np.empty(len(col), dtype=object)
+            out[name][:] = col.tolist()
+    return out
+
+
 def to_df(
     table: Table,
     *,
@@ -195,6 +213,9 @@ def to_df(
 
     # Encode mixins and validate columns
     tbl = _encode_mixins(table)
+    # Support numpy multidimensional arrays in columns for pandas-like backend
+    if backend_impl.is_pandas_like():
+        tbl = _convert_multidim_columns_to_object(tbl)
     _validate_columns_for_backend(tbl, backend_impl=backend_impl)
 
     # Convert to narwhals DataFrame
@@ -384,6 +405,8 @@ def to_pandas(
 
     # Encode mixins and validate columns
     tbl = _encode_mixins(table)
+    # Support numpy multidimensional arrays in columns
+    tbl = _convert_multidim_columns_to_object(tbl)
     _validate_columns_for_backend(tbl, backend_impl=PANDAS_LIKE)  # pandas validation
 
     out = {}


Test patch:
diff --git a/astropy/table/tests/test_df.py b/astropy/table/tests/test_df.py
index f4d5c2ceb9f5..27479da1b893 100644
--- a/astropy/table/tests/test_df.py
+++ b/astropy/table/tests/test_df.py
@@ -446,8 +446,14 @@ def test_nd_columns(self, backend, use_legacy_pandas_api, ndim):
         t["a"] = np.arange(np.prod(colshape)).reshape(colshape)
 
         match backend:
-            # Pandas and PyArrow do not support multidimensional columns
-            case "pandas" | "pyarrow":
+            # Pandas support multidimensional columns
+            case "pandas":
+                if ndim > 1:
+                    df = self._to_dataframe(t, backend, use_legacy_pandas_api)
+                    assert hasattr(df["a"].iloc[0], "__len__")
+                    return
+            # PyArrow do not support multidimensional columns
+            case "pyarrow":
                 if ndim > 1:
                     with pytest.raises(
                         ValueError,
@@ -678,3 +684,49 @@ def test_from_pandas_df_with_qtable(method):
     df = t.to_pandas()
     qt = getattr(table.QTable, method)(df)
     assert isinstance(qt, table.QTable)
+
+
+@pytest.mark.skipif(not HAS_PANDAS, reason="require pandas")
+@pytest.mark.parametrize("use_legacy_pandas_api", [True, False])
+def test_pandas_conversion_multidim_columns(use_legacy_pandas_api):
+    """Test that Table with multidim columns converts successfully to pandas (#19173).
+
+    This test only uses pandas since other backends do not support multidim columns.
+    It includes a variety of column types and dimensions (1d to 3d), including
+    masked columns to verify that all are handled correctly.
+    """
+    if not use_legacy_pandas_api and not HAS_NARWHALS:
+        pytest.skip("requires narwhals for generic pandas conversion")
+
+    t = Table()
+    t["a"] = ["foo", "bar"]  # 1-d str
+    t["b"] = [1.5, 2.5]  # 1-d float
+    t["c"] = np.array([[1, 2], [3, 4]])  # 2-d int
+    t["d"] = np.array([[[1.5, 0], [2, 1]], [[3, 2], [4, 3]]])  # 3-d float
+    t["e"] = np.array([["a", "b"], ["c", "d"]])  # 2-d str
+    t["f"] = np.empty((2, 2), dtype=object)
+    t["f"][:] = [[None, {"a": 1}], ["str", [1, 2]]]  # 2-d object type
+    t["g"] = np.ma.MaskedArray([[1, 2], [3, 4]], mask=[[True, False], [False, True]])
+    t["h"] = np.ma.MaskedArray([[1.5, 2], [3, 4]], mask=[[True, False], [False, True]])
+    tc = t.copy()
+    df = t.to_pandas() if use_legacy_pandas_api else t.to_df("pandas")
+
+    # Ensure that conversion process leaves `t` unchanged
+    assert np.all(t == tc)
+
+    # Check properties of converted dataframe `df`:
+    # - dtypes are as expected (all object except float64 for column "b").
+    # - Dataframe values exactly match table column values via .tolist().
+    # - All dataframe Series items for multidim columns are type list.
+    for name in t.colnames:
+        assert df[name].dtype == "float64" if name == "b" else "object"
+        assert df[name].tolist() == t[name].tolist()
+        if t[name].ndim > 1:
+            for val in df[name]:
+                assert type(val) is list
+
+    # Special-case testing for masked column
+    assert df["g"][0] == [None, 2]
+    assert df["g"][1] == [3, None]
+    assert df["h"][0] == [None, 2.0]
+    assert df["h"][1] == [3.0, None]