Repository: scikit-hep/awkward

Issue:
Support `pyarrow.array(ak.Array(...))`
### Description of new feature

Arrow provides an `__arrow_array__` interface that we *could* support.

@jpivarski how do you feel about this? The questions that arise for me are - what defaults should we choose for `ak.to_arrow`? I'm mainly concerned about enabling the extension array or not. To my mind, the extension-array mechanism is useful for serialisation, but may cause problems for strict Arrow routines.


Discussion:
I'm in favor, and I think it makes sense for `__arrow_array__` to invoke `ak.to_arrow` with `extensionarray=False`.

Resolving patch:
diff --git a/src/awkward/_connect/pyarrow.py b/src/awkward/_connect/pyarrow.py
--- a/src/awkward/_connect/pyarrow.py
+++ b/src/awkward/_connect/pyarrow.py
@@ -1114,3 +1114,11 @@ def form_handle_arrow(schema, pass_empty_field=False):
 
         else:
             return out
+
+
+def convert_to_array(layout, type=None):
+    out = ak.operations.to_arrow(layout, extensionarray=False)
+    if type is None:
+        return out
+    else:
+        return pyarrow.array(out, type=type)
diff --git a/src/awkward/highlevel.py b/src/awkward/highlevel.py
--- a/src/awkward/highlevel.py
+++ b/src/awkward/highlevel.py
@@ -3,6 +3,7 @@
 
 __all__ = ("Array", "ArrayBuilder", "Record")
 
+import builtins
 import copy
 import html
 import io
@@ -1325,6 +1326,14 @@ def __array__(self, *args, **kwargs):
         with ak._errors.OperationErrorContext("numpy.asarray", (self, *args), kwargs):
             return ak._connect.numpy.convert_to_array(self._layout, args, kwargs)
 
+    def __arrow_array__(self, type=None):
+        with ak._errors.OperationErrorContext(
+            f"{builtins.type(self).__name__}.__arrow_array__", (self,), {"type": type}
+        ):
+            from awkward._connect.pyarrow import convert_to_array
+
+            return convert_to_array(self._layout, type=type)
+
     def __array_ufunc__(self, ufunc, method, *inputs, **kwargs):
         """
         Intercepts attempts to pass this Array to a NumPy
@@ -2417,6 +2426,14 @@ def __array__(self, *args, **kwargs):
         with ak._errors.OperationErrorContext("numpy.asarray", (self, *args), kwargs):
             return ak._connect.numpy.convert_to_array(self.snapshot(), args, kwargs)
 
+    def __arrow_array__(self, type=None):
+        with ak._errors.OperationErrorContext(
+            f"{builtins.type(self).__name__}.__arrow_array__", (self,), {"type": type}
+        ):
+            from awkward._connect.pyarrow import convert_to_array
+
+            return convert_to_array(self.snapshot(), type=type)
+
     @property
     def numba_type(self):
         """
diff --git a/src/awkward/operations/str/__init__.py b/src/awkward/operations/str/__init__.py
--- a/src/awkward/operations/str/__init__.py
+++ b/src/awkward/operations/str/__init__.py
@@ -119,6 +119,9 @@ def action(layout, **absorb):
                     ),
                     highlevel=False,
                 )
+
+                if out.is_option:
+                    out = out.content
                 if out.is_list and out.parameter("__array__") == "string":
                     out = out.copy(
                         content=out.content.copy(parameters={"__array__": "byte"}),
@@ -140,6 +143,9 @@ def action(layout, **absorb):
 def _erase_list_option(layout):
     from awkward.contents.unmaskedarray import UnmaskedArray
 
+    if layout.is_option:
+        layout = layout.content
+
     assert layout.is_list
     if layout.content.is_option:
         assert isinstance(layout.content, UnmaskedArray)


Test patch:
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -187,9 +187,15 @@ jobs:
           - '3.8'
         numpy-package:
           - "numpy"
+        pyarrow-package:
+          - "pyarrow"
         include:
           - python-version: '3.8'
             numpy-package: "numpy==1.18.0"
+            pyarrow-package: "pyarrow"
+          - python-version: '3.8'
+            numpy-package: "numpy"
+            pyarrow-package: "pyarrow==7.0.0"
 
     runs-on: ubuntu-20.04
 
@@ -223,7 +229,7 @@ jobs:
           python -m build -w ./awkward-cpp
 
       - name: Install awkward-cpp
-        run: python -m pip install -v ./awkward-cpp/dist/*.whl "${{ matrix.numpy-package }}"
+        run: python -m pip install -v ./awkward-cpp/dist/*.whl "${{ matrix.numpy-package }}" "${{ matrix.pyarrow-package }}"
 
       - name: Build & install awkward
         run: python -m pip install -v .
diff --git a/tests/test_2650_arrow_array_highlevel.py b/tests/test_2650_arrow_array_highlevel.py
new file mode 100644
--- /dev/null
+++ b/tests/test_2650_arrow_array_highlevel.py
@@ -0,0 +1,46 @@
+# BSD 3-Clause License; see https://github.com/scikit-hep/awkward-1.0/blob/main/LICENSE
+
+import numpy as np
+import pytest
+from packaging.version import parse as parse_version
+
+import awkward as ak
+
+pyarrow = pytest.importorskip("pyarrow")
+pyarrow_types = pytest.importorskip("pyarrow.types")
+
+
+def test_simple_conversion():
+    array = ak.Array(
+        ak.contents.RecordArray(
+            [
+                ak.to_layout(np.arange(6, dtype=np.uint8)),
+                ak.to_layout(np.arange(6, dtype=np.float64)),
+            ],
+            ["x", "y"],
+        )
+    )
+    arrow_array = pyarrow.array(array)
+
+    assert arrow_array.tolist() == array.to_list()
+
+
+@pytest.mark.skipif(
+    parse_version(pyarrow.__version__) < parse_version("12.0.0"),
+    reason="pyarrow >= 12.0.0 required for casting test",
+)
+def test_type_cast():
+    array = ak.mask(np.array([1, 2, 3], dtype=np.uint8), [True, False, False])
+    arrow_array = pyarrow.array(array, type=pyarrow.float64())
+    assert pyarrow_types.is_float64(arrow_array.type)
+
+    arrow_array = pyarrow.array(array)
+    assert pyarrow_types.is_uint8(arrow_array.type)
+
+
+def test_zero_copy():
+    np_array = np.array([1, 2, 3], dtype=np.uint32)
+    ak_array = ak.from_numpy(np_array)
+    arrow_array = pyarrow.array(ak_array)
+    assert pyarrow_types.is_uint32(arrow_array.type)
+    assert np.shares_memory(np_array, arrow_array.to_numpy())