Repository: scikit-hep/awkward

Issue:
`ak.from_arrow` broken for ChunkedArrays
### Version of Awkward Array

2.5.0

### Description and code to reproduce

Tested with awkward 2.4.6 through 2.5.1rc1 and PyArrow 13.0.0 through 14.0.1. Conda environment created with, e.g., `mamba install -y jupyterlab awkward=2.5.0 pyarrow=14.0.1`.

```python
import pyarrow as pa
import awkward as ak
ary = pa.chunked_array([["foo", "bar"], ["blah", "bleh"]])
ak.from_arrow(ary)
```
results in `AttributeError: 'UnmaskedArray' object has no attribute '__pyarrow_original'`.

<details>

```
---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
Cell In[12], line 1
----> 1 ak.from_arrow(ary)

File ~/micromamba/envs/test/lib/python3.12/site-packages/awkward/_dispatch.py:39, in named_high_level_function.<locals>.dispatch(*args, **kwargs)
     35 @wraps(func)
     36 def dispatch(*args, **kwargs):
     37     # NOTE: this decorator assumes that the operation is exposed under `ak.`
     38     with OperationErrorContext(name, args, kwargs):
---> 39         gen_or_result = func(*args, **kwargs)
     40         if isgenerator(gen_or_result):
     41             array_likes = next(gen_or_result)

File ~/micromamba/envs/test/lib/python3.12/site-packages/awkward/operations/ak_from_arrow.py:45, in from_arrow(array, generate_bitmasks, highlevel, behavior, attrs)
     15 @high_level_function()
     16 def from_arrow(
     17     array, *, generate_bitmasks=False, highlevel=True, behavior=None, attrs=None
     18 ):
     19     """
     20     Args:
     21         array (`pyarrow.Array`, `pyarrow.ChunkedArray`, `pyarrow.RecordBatch`, or `pyarrow.Table`):
   (...)
     43     See also #ak.to_arrow, #ak.to_arrow_table, #ak.from_parquet, #ak.from_arrow_schema.
     44     """
---> 45     return _impl(array, generate_bitmasks, highlevel, behavior, attrs)

File ~/micromamba/envs/test/lib/python3.12/site-packages/awkward/operations/ak_from_arrow.py:67, in _impl(array, generate_bitmasks, highlevel, behavior, attrs)
     65 if awkwardarrow_type is None:
     66     if isinstance(out, ak.contents.UnmaskedArray):
---> 67         out = awkward._connect.pyarrow.remove_optiontype(out)
     68 else:
     69     if awkwardarrow_type.mask_type in (None, "IndexedArray"):

File ~/micromamba/envs/test/lib/python3.12/site-packages/awkward/_connect/pyarrow.py:913, in remove_optiontype(akarray)
    912 def remove_optiontype(akarray):
--> 913     return akarray.__pyarrow_original

AttributeError: 'UnmaskedArray' object has no attribute '__pyarrow_original'

This error occurred while calling

    ak.from_arrow(
        ChunkedArray-instance
    )
```

</details>

```python
import pyarrow as pa
import awkward as ak
ary = pa.chunked_array([[1,2,3], [4,5]])
ak.from_arrow(ary)
```
results in `ModuleNotFoundError: No module named 'pandas'`.

<details>

```
---------------------------------------------------------------------------
ModuleNotFoundError                       Traceback (most recent call last)
Cell In[2], line 5
      3 #ary = pa.chunked_array([["foo", "bar"], ["blah", "bleh"]])
      4 ary = pa.chunked_array([[1,2,3], [4,5]])
----> 5 ak.from_arrow(ary)

File ~/micromamba/envs/test/lib/python3.12/site-packages/awkward/_dispatch.py:39, in named_high_level_function.<locals>.dispatch(*args, **kwargs)
     35 @wraps(func)
     36 def dispatch(*args, **kwargs):
     37     # NOTE: this decorator assumes that the operation is exposed under `ak.`
     38     with OperationErrorContext(name, args, kwargs):
---> 39         gen_or_result = func(*args, **kwargs)
     40         if isgenerator(gen_or_result):
     41             array_likes = next(gen_or_result)

File ~/micromamba/envs/test/lib/python3.12/site-packages/awkward/operations/ak_from_arrow.py:45, in from_arrow(array, generate_bitmasks, highlevel, behavior, attrs)
     15 @high_level_function()
     16 def from_arrow(
     17     array, *, generate_bitmasks=False, highlevel=True, behavior=None, attrs=None
     18 ):
     19     """
     20     Args:
     21         array (`pyarrow.Array`, `pyarrow.ChunkedArray`, `pyarrow.RecordBatch`, or `pyarrow.Table`):
   (...)
     43     See also #ak.to_arrow, #ak.to_arrow_table, #ak.from_parquet, #ak.from_arrow_schema.
     44     """
---> 45     return _impl(array, generate_bitmasks, highlevel, behavior, attrs)

File ~/micromamba/envs/test/lib/python3.12/site-packages/awkward/operations/ak_from_arrow.py:55, in _impl(array, generate_bitmasks, highlevel, behavior, attrs)
     51 pyarrow = awkward._connect.pyarrow.pyarrow
     53 ctx = HighLevelContext(behavior=behavior, attrs=attrs).finalize()
---> 55 out = awkward._connect.pyarrow.handle_arrow(
     56     array, generate_bitmasks=generate_bitmasks, pass_empty_field=True
     57 )
     59 if isinstance(array, (pyarrow.lib.Array, pyarrow.lib.ChunkedArray)):
     60     (
     61         awkwardarrow_type,
     62         storage_type,
     63     ) = awkward._connect.pyarrow.to_awkwardarrow_storage_types(array.type)

File ~/micromamba/envs/test/lib/python3.12/site-packages/awkward/_connect/pyarrow.py:933, in handle_arrow(obj, generate_bitmasks, pass_empty_field)
    930     return out
    932 elif isinstance(obj, pyarrow.lib.ChunkedArray):
--> 933     layouts = [handle_arrow(x, generate_bitmasks) for x in obj.chunks if len(x) > 0]
    935     if len(layouts) == 1:
    936         return layouts[0]

File ~/micromamba/envs/test/lib/python3.12/site-packages/awkward/_connect/pyarrow.py:926, in handle_arrow(obj, generate_bitmasks, pass_empty_field)
    922 buffers = obj.buffers()
    924 awkwardarrow_type, storage_type = to_awkwardarrow_storage_types(obj.type)
--> 926 out = popbuffers(
    927     obj, awkwardarrow_type, storage_type, buffers, generate_bitmasks
    928 )
    929 assert len(buffers) == 0
    930 return out

File ~/micromamba/envs/test/lib/python3.12/site-packages/awkward/_connect/pyarrow.py:667, in popbuffers(paarray, awkwardarrow_type, storage_type, buffers, generate_bitmasks)
    665     data = numpy.astype(numpy.frombuffer(data, dtype=np.int32), dtype=np.int64)
    666 if dt is None:
--> 667     dt = storage_type.to_pandas_dtype()
    669 out = ak.contents.NumpyArray(
    670     numpy.frombuffer(data, dtype=dt),
    671     parameters=node_parameters(awkwardarrow_type),
    672     backend=NumpyBackend.instance(),
    673 )
    674 return popbuffers_finalize(
    675     out, paarray, validbits, awkwardarrow_type, generate_bitmasks
    676 )

File ~/micromamba/envs/test/lib/python3.12/site-packages/pyarrow/types.pxi:336, in pyarrow.lib.DataType.to_pandas_dtype()

File ~/micromamba/envs/test/lib/python3.12/site-packages/pyarrow/types.pxi:154, in pyarrow.lib._to_pandas_dtype()

File ~/micromamba/envs/test/lib/python3.12/site-packages/pyarrow/pandas-shim.pxi:158, in pyarrow.lib._PandasAPIShim.is_v1()

File ~/micromamba/envs/test/lib/python3.12/site-packages/pyarrow/pandas-shim.pxi:100, in pyarrow.lib._PandasAPIShim._check_import()

File ~/micromamba/envs/test/lib/python3.12/site-packages/pyarrow/pandas-shim.pxi:53, in pyarrow.lib._PandasAPIShim._import_pandas()

File ~/micromamba/envs/test/lib/python3.12/site-packages/pyarrow/pandas-shim.pxi:48, in pyarrow.lib._PandasAPIShim._import_pandas()

ModuleNotFoundError: No module named 'pandas'

This error occurred while calling

    ak.from_arrow(
        ChunkedArray-instance
    )
```

</details>

For the integer ChunkedArray case, if I install pandas, then I get the same error as for the string ChunkedArray.

For both integer and string arrays, `ak.from_arrow(ary.combine_chunks())` works.


Discussion:

Resolving patch:
diff --git a/src/awkward/_connect/pyarrow.py b/src/awkward/_connect/pyarrow.py
--- a/src/awkward/_connect/pyarrow.py
+++ b/src/awkward/_connect/pyarrow.py
@@ -919,8 +919,15 @@ def direct_Content_subclass_name(node):
         return out.__name__
 
 
+def is_revertable(akarray):
+    return hasattr(akarray, "__pyarrow_original")
+
+
 def remove_optiontype(akarray):
-    return akarray.__pyarrow_original
+    if callable(akarray.__pyarrow_original):
+        return akarray.__pyarrow_original()
+    else:
+        return akarray.__pyarrow_original
 
 
 def form_remove_optiontype(akform):
@@ -944,6 +951,17 @@ def handle_arrow(obj, generate_bitmasks=False, pass_empty_field=False):
 
         if len(layouts) == 1:
             return layouts[0]
+        elif any(is_revertable(arr) for arr in layouts):
+            assert all(is_revertable(arr) for arr in layouts)
+            # TODO: the callable argument to revertable is a premature(?) optimisation.
+            #       it would be better to obviate the need to compute both revertable and non revertable branches
+            #       e.g. by requesting a particular layout kind from the next `frombuffers` operation
+            return revertable(
+                ak.operations.concatenate(layouts, highlevel=False),
+                lambda: ak.operations.concatenate(
+                    [remove_optiontype(x) for x in layouts], highlevel=False
+                ),
+            )
         else:
             return ak.operations.concatenate(layouts, highlevel=False)
 
@@ -1044,7 +1062,19 @@ def handle_arrow(obj, generate_bitmasks=False, pass_empty_field=False):
                 for batch in batches
                 if len(batch) > 0
             ]
-            return ak.operations.concatenate(arrays, highlevel=False)
+            if any(is_revertable(arr) for arr in arrays):
+                assert all(is_revertable(arr) for arr in arrays)
+                # TODO: the callable argument to revertable is a premature(?) optimisation.
+                #       it would be better to obviate the need to compute both revertable and non revertable branches
+                #       e.g. by requesting a particular layout kind from the next `frombuffers` operation
+                return revertable(
+                    ak.operations.concatenate(arrays, highlevel=False),
+                    lambda: ak.operations.concatenate(
+                        [remove_optiontype(x) for x in arrays], highlevel=False
+                    ),
+                )
+            else:
+                return ak.operations.concatenate(arrays, highlevel=False)
 
     elif (
         isinstance(obj, Iterable)


Test patch:
diff --git a/tests/test_2889_test_chunked_array.py b/tests/test_2889_test_chunked_array.py
new file mode 100644
--- /dev/null
+++ b/tests/test_2889_test_chunked_array.py
@@ -0,0 +1,49 @@
+# BSD 3-Clause License; see https://github.com/scikit-hep/awkward/blob/main/LICENSE
+
+from __future__ import annotations
+
+import pytest
+
+import awkward as ak
+
+pa = pytest.importorskip("pyarrow")
+
+
+def test_strings():
+    array = pa.chunked_array([["foo", "bar"], ["blah", "bleh"]])
+    ak_array = ak.from_arrow(array)
+    assert ak_array.type == ak.types.ArrayType(
+        ak.types.ListType(
+            ak.types.NumpyType("uint8", parameters={"__array__": "char"}),
+            parameters={"__array__": "string"},
+        ),
+        4,
+    )
+
+
+def test_strings_option():
+    array = pa.chunked_array([["foo", "bar"], ["blah", "bleh", None]])
+    ak_array = ak.from_arrow(array)
+    assert ak_array.type == ak.types.ArrayType(
+        ak.types.OptionType(
+            ak.types.ListType(
+                ak.types.NumpyType("uint8", parameters={"__array__": "char"}),
+                parameters={"__array__": "string"},
+            )
+        ),
+        5,
+    )
+
+
+def test_numbers():
+    array = pa.chunked_array([[1, 2, 3], [4, 5]])
+    ak_array = ak.from_arrow(array)
+    assert ak_array.type == ak.types.ArrayType(ak.types.NumpyType("int64"), 5)
+
+
+def test_numbers_option():
+    array = pa.chunked_array([[1, 2, 3], [4, 5, None]])
+    ak_array = ak.from_arrow(array)
+    assert ak_array.type == ak.types.ArrayType(
+        ak.types.OptionType(ak.types.NumpyType("int64")), 6
+    )