Repository: lincc-frameworks/nested-pandas

Issue:
Cannot convert pyarrow.lib.ChunkedArray to pyarrow.lib.Array at view_sorted_series_as_list_array
**Bug report**

Another `Cannot convert pyarrow.lib.ChunkedArray to pyarrow.lib.Array` error, caught when working with ComCam data
```
File ~/.local/lib/python3.11/site-packages/nested_pandas/series/packer.py:272, in view_sorted_series_as_list_array(series, offset, unique_index)
    [269](https://vscode-remote+ssh-002dremote-002bslac-002ddev.vscode-resource.vscode-cdn.net/sdf/home/k/kostya/rubin-user/linccf/single_object/~/.local/lib/python3.11/site-packages/nested_pandas/series/packer.py:269) if unique_index is None:
    [270](https://vscode-remote+ssh-002dremote-002bslac-002ddev.vscode-resource.vscode-cdn.net/sdf/home/k/kostya/rubin-user/linccf/single_object/~/.local/lib/python3.11/site-packages/nested_pandas/series/packer.py:270)     unique_index = series.index[offset[:-1]]
--> [272](https://vscode-remote+ssh-002dremote-002bslac-002ddev.vscode-resource.vscode-cdn.net/sdf/home/k/kostya/rubin-user/linccf/single_object/~/.local/lib/python3.11/site-packages/nested_pandas/series/packer.py:272) list_array = pa.ListArray.from_arrays(
    [273](https://vscode-remote+ssh-002dremote-002bslac-002ddev.vscode-resource.vscode-cdn.net/sdf/home/k/kostya/rubin-user/linccf/single_object/~/.local/lib/python3.11/site-packages/nested_pandas/series/packer.py:273)     offset,
    [274](https://vscode-remote+ssh-002dremote-002bslac-002ddev.vscode-resource.vscode-cdn.net/sdf/home/k/kostya/rubin-user/linccf/single_object/~/.local/lib/python3.11/site-packages/nested_pandas/series/packer.py:274)     pa.array(series, from_pandas=True),
    [275](https://vscode-remote+ssh-002dremote-002bslac-002ddev.vscode-resource.vscode-cdn.net/sdf/home/k/kostya/rubin-user/linccf/single_object/~/.local/lib/python3.11/site-packages/nested_pandas/series/packer.py:275) )
    [276](https://vscode-remote+ssh-002dremote-002bslac-002ddev.vscode-resource.vscode-cdn.net/sdf/home/k/kostya/rubin-user/linccf/single_object/~/.local/lib/python3.11/site-packages/nested_pandas/series/packer.py:276) return pd.Series(
    [277](https://vscode-remote+ssh-002dremote-002bslac-002ddev.vscode-resource.vscode-cdn.net/sdf/home/k/kostya/rubin-user/linccf/single_object/~/.local/lib/python3.11/site-packages/nested_pandas/series/packer.py:277)     list_array,
    [278](https://vscode-remote+ssh-002dremote-002bslac-002ddev.vscode-resource.vscode-cdn.net/sdf/home/k/kostya/rubin-user/linccf/single_object/~/.local/lib/python3.11/site-packages/nested_pandas/series/packer.py:278)     dtype=pd.ArrowDtype(list_array.type),
   (...)
    [281](https://vscode-remote+ssh-002dremote-002bslac-002ddev.vscode-resource.vscode-cdn.net/sdf/home/k/kostya/rubin-user/linccf/single_object/~/.local/lib/python3.11/site-packages/nested_pandas/series/packer.py:281)     name=series.name,
    [282](https://vscode-remote+ssh-002dremote-002bslac-002ddev.vscode-resource.vscode-cdn.net/sdf/home/k/kostya/rubin-user/linccf/single_object/~/.local/lib/python3.11/site-packages/nested_pandas/series/packer.py:282) )

File /sdf/group/rubin/sw/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/pyarrow/array.pxi:2270, in pyarrow.lib.ListArray.from_arrays()

TypeError: Cannot convert pyarrow.lib.ChunkedArray to pyarrow.lib.Array
```

**Before submitting**
Please check the following:

- [x] I have described the situation in which the bug arose, including what code was executed, information about my environment, and any applicable data others will need to reproduce the problem.
- [ ] I have included available evidence of the unexpected behavior (including error messages, screenshots, and/or plots) as well as a description of what I expected instead.
- [ ] If I have a solution in mind, I have provided an explanation and/or pseudocode and/or task list.


Discussion:

Fix (patch):
diff --git a/src/nested_pandas/series/packer.py b/src/nested_pandas/series/packer.py
index dedaaca..73dc58c 100644
--- a/src/nested_pandas/series/packer.py
+++ b/src/nested_pandas/series/packer.py
@@ -293,10 +293,17 @@ def view_sorted_series_as_list_array(
     if unique_index is None:
         unique_index = series.index[offset[:-1]]
 
+    # Input series may be represented by pyarrow.ChunkedArray, in this case pa.array(series) would fail
+    # with "TypeError: Cannot convert a 'ChunkedArray' to a 'ListArray'".
+    # https://github.com/lincc-frameworks/nested-pandas/issues/189
+    flat_array = pa.array(series, from_pandas=True)
+    if isinstance(flat_array, pa.ChunkedArray):
+        flat_array = flat_array.combine_chunks()
     list_array = pa.ListArray.from_arrays(
         offset,
-        pa.array(series, from_pandas=True),
+        flat_array,
     )
+
     return pd.Series(
         list_array,
         dtype=pd.ArrowDtype(list_array.type),


Test changes:
diff --git a/tests/nested_pandas/series/test_packer.py b/tests/nested_pandas/series/test_packer.py
index b022a17..df59c0f 100644
--- a/tests/nested_pandas/series/test_packer.py
+++ b/tests/nested_pandas/series/test_packer.py
@@ -524,6 +524,30 @@ def test_view_sorted_series_as_list_array_raises_when_not_sorted():
         packer.view_sorted_series_as_list_array(series)
 
 
+def test_view_sorted_series_as_list_array_chunked_input():
+    """Issue #189
+
+    https://github.com/lincc-frameworks/nested-pandas/issues/189
+    """
+    series = pd.Series(
+        pa.chunked_array([pa.array([0, 1, 2]), pa.array([None, 4])]),
+        name="a",
+        index=np.arange(5),
+        dtype=pd.ArrowDtype(pa.int64()),
+    )
+    offset = np.array([0, 2, 4, 5])
+    unique_index = ["x", "y", "z"]
+    desired = pd.Series(
+        pa.array([[0, 1], [2, None], [4]]),
+        index=unique_index,
+        dtype=pd.ArrowDtype(pa.list_(pa.int64())),
+        name="a",
+    )
+
+    actual = packer.view_sorted_series_as_list_array(series, offset, unique_index)
+    assert_series_equal(actual, desired)
+
+
 @pytest.mark.parametrize(
     "index,offsets",
     [