Repository: scikit-hep/awkward

Issue:
NEP18-dispatched functions raise an error if any arguments are non-native endian arrays
### Version of Awkward Array

HEAD

### Description and code to reproduce

We can pass, for example, an Awkward Array and a NumPy array to `np.array_equal` (without having an explicit implementation of `ak.array_equal`) because we're overloading it with NEP-18 (`__array_function__`).

```python
>>> np.array_equal(ak.Array(np.arange(10, dtype="<u4")), np.arange(10, dtype="<u4"))
True
```

This function is supposed to be insensitive to the dtypes of its arguments.

```python
>>> np.array_equal(ak.Array(np.arange(10, dtype="<u4")), np.arange(10, dtype="<f4"))
True
```

Including endianness.

```
>>> np.array_equal(np.arange(10, dtype="<u4"), np.arange(10, dtype=">u4"))
True
```

But this fails:

```python
>>> np.array_equal(ak.Array(np.arange(10, dtype="<u4")), np.arange(10, dtype=">u4"))
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "<__array_function__ internals>", line 180, in array_equal
  File "/home/jpivarski/irishep/awkward/src/awkward/highlevel.py", line 1392, in __array_function__
    return ak._connect.numpy.array_function(
  File "/home/jpivarski/irishep/awkward/src/awkward/_connect/numpy.py", line 73, in array_function
    return _array_function_no_impl(func, types, args, kwargs, behavior)
  File "/home/jpivarski/irishep/awkward/src/awkward/_connect/numpy.py", line 59, in _array_function_no_impl
    rectilinear_args = tuple(_to_rectilinear(x) for x in args)
  File "/home/jpivarski/irishep/awkward/src/awkward/_connect/numpy.py", line 59, in <genexpr>
    rectilinear_args = tuple(_to_rectilinear(x) for x in args)
  File "/home/jpivarski/irishep/awkward/src/awkward/_connect/numpy.py", line 42, in _to_rectilinear
    return ak.operations.to_numpy(arg)
  File "/home/jpivarski/irishep/awkward/src/awkward/operations/ak_to_numpy.py", line 41, in to_numpy
    return _impl(array, allow_missing)
  File "/home/jpivarski/irishep/awkward/src/awkward/operations/ak_to_numpy.py", line 48, in _impl
    layout = ak.to_layout(array, allow_record=False)
  File "/home/jpivarski/irishep/awkward/src/awkward/operations/ak_to_layout.py", line 51, in to_layout
    return _impl(array, allow_record, allow_other, regulararray=regulararray)
  File "/home/jpivarski/irishep/awkward/src/awkward/operations/ak_to_layout.py", line 84, in _impl
    return ak.operations.from_numpy(
  File "/home/jpivarski/irishep/awkward/src/awkward/operations/ak_from_numpy.py", line 50, in from_numpy
    return ak._util.from_arraylib(
  File "/home/jpivarski/irishep/awkward/src/awkward/_util.py", line 709, in from_arraylib
    layout = recurse(array, mask)
  File "/home/jpivarski/irishep/awkward/src/awkward/_util.py", line 667, in recurse
    data = ak.contents.NumpyArray(array)
  File "/home/jpivarski/irishep/awkward/src/awkward/contents/numpyarray.py", line 95, in __init__
    ak.types.numpytype.dtype_to_primitive(self._data.dtype)
  File "/home/jpivarski/irishep/awkward/src/awkward/types/numpytype.py", line 49, in dtype_to_primitive
    raise ak._errors.wrap_error(
TypeError: while calling

    ak.to_numpy(
        array = numpy.ndarray([0 1 2 ... 7 8 9])
        allow_missing = True
    )

Error details: unsupported dtype: dtype('>u4'). Must be one of

    bool, int8, uint8, int16, uint16, int32, uint32, int64, uint64, float32, float64, complex64, complex128, datetime64, timedelta64, float16, float128, complex256

or a datetime64/timedelta64 with units (e.g. 'datetime64[15us]')
```

because somewhere in the implementation of `ak.Array.__array_function__`, we try to convert the non-native endian NumPy array into an Awkward Array, and we don't support that.

The implementation will have to become insensitive to this, possibly by ensuring that all non-Awkward arguments in `__array_function__` are converted to native endian, or maybe by taking a new policy that non-native endian NumPy arrays are copy-converted to native endian in `to_layout` and/or `from_numpy`. That would introduce a hidden cost for non-native endian arrays, but I think that would be acceptable. It's not like GPU copying, where users really want to be aware of this; endianness is a nuisance, rather than something people want to track.


Discussion:

Resolving patch:
diff --git a/src/awkward/_backends/dispatch.py b/src/awkward/_backends/dispatch.py
--- a/src/awkward/_backends/dispatch.py
+++ b/src/awkward/_backends/dispatch.py
@@ -1,7 +1,7 @@
 # BSD 3-Clause License; see https://github.com/scikit-hep/awkward-1.0/blob/main/LICENSE
 from __future__ import annotations
 
-from collections.abc import Collection
+from collections.abc import Iterable
 
 from awkward._backends.backend import Backend
 from awkward._nplikes.numpy import Numpy
@@ -46,7 +46,7 @@ def lookup_factory(cls):
     return wrapper
 
 
-def common_backend(backends: Collection[Backend]) -> Backend:
+def common_backend(backends: Iterable[Backend]) -> Backend:
     unique_backends = frozenset(backends)
     # Either we have one nplike, or one + typetracer
     if len(unique_backends) == 1:
@@ -57,10 +57,16 @@ def common_backend(backends: Collection[Backend]) -> Backend:
             if not backend.nplike.known_data:
                 return backend
 
-        raise ValueError(
-            "cannot operate on arrays with incompatible backends. Use #ak.to_backend to coerce the arrays "
-            "to the same backend"
-        )
+        if len(unique_backends) > 1:
+            raise ValueError(
+                "cannot operate on arrays with incompatible backends. Use #ak.to_backend to coerce the arrays "
+                "to the same backend"
+            )
+
+        else:
+            raise ValueError(
+                "no backends were given in order to determine a common backend."
+            )
 
 
 def _backend_of(obj, default: D = unset) -> Backend | D:
diff --git a/src/awkward/_connect/numpy.py b/src/awkward/_connect/numpy.py
--- a/src/awkward/_connect/numpy.py
+++ b/src/awkward/_connect/numpy.py
@@ -1,11 +1,15 @@
 # BSD 3-Clause License; see https://github.com/scikit-hep/awkward-1.0/blob/main/LICENSE
+import collections
 import functools
 import inspect
+from collections.abc import Iterable
+from itertools import chain
 
 import numpy
 
 import awkward as ak
-from awkward._backends.dispatch import backend_of
+from awkward._backends.backend import Backend
+from awkward._backends.dispatch import backend_of, common_backend
 from awkward._behavior import (
     behavior_of,
     find_custom_cast,
@@ -13,7 +17,9 @@
     find_ufunc_generic,
 )
 from awkward._layout import wrap_layout
+from awkward._nplikes import to_nplike
 from awkward._regularize import is_non_string_like_iterable
+from awkward._typing import Iterator
 from awkward._util import numpy_at_least
 from awkward.contents.numpyarray import NumpyArray
 
@@ -44,15 +50,45 @@ def convert_to_array(layout, args, kwargs):
 implemented = {}
 
 
-def _to_rectilinear(arg):
-    backend = backend_of(arg, default=None)
-    # We have some array-like object that our backend mechanism understands
-    if backend is not None:
-        return ak.operations.to_numpy(arg)
+def _find_backends(args: Iterable) -> Iterator[Backend]:
+    """
+    Args:
+        args: iterable of objects to visit
+
+    Yields the encountered backends of layout / array-like arguments encountered
+    in the argument list.
+    """
+    stack = collections.deque(args)
+    while stack:
+        arg = stack.popleft()
+        # If the argument declares a backend, easy!
+        backend = backend_of(arg, default=None)
+        if backend is not None:
+            yield backend
+        # Otherwise, traverse into supported sequence types
+        elif isinstance(arg, (tuple, list)):
+            stack.extend(arg)
+
+
+def _to_rectilinear(arg, backend: Backend):
+    # Is this object something we already associate with a backend?
+    arg_backend = backend_of(arg, default=None)
+    if arg_backend is not None:
+        arg_nplike = arg_backend.nplike
+        # Is this argument already in a backend-supported form?
+        if arg_nplike.is_own_array(arg):
+            # Convert to the appropriate nplike
+            return to_nplike(
+                arg_nplike.asarray(arg), backend.nplike, from_nplike=arg_nplike
+            )
+        # Otherwise, cast to layout and convert
+        else:
+            layout = ak.to_layout(arg, allow_record=False, allow_other=False)
+            return layout.to_backend(backend).to_backend_array(allow_missing=True)
     elif isinstance(arg, tuple):
-        return tuple(_to_rectilinear(x) for x in arg)
+        return tuple(_to_rectilinear(x, backend) for x in arg)
     elif isinstance(arg, list):
-        return [_to_rectilinear(x) for x in arg]
+        return [_to_rectilinear(x, backend) for x in arg]
     elif is_non_string_like_iterable(arg):
         raise TypeError(
             f"encountered an unsupported iterable value {arg!r} whilst converting arguments to NumPy-friendly "
@@ -63,8 +99,10 @@ def _to_rectilinear(arg):
 
 
 def _array_function_no_impl(func, types, args, kwargs, behavior):
-    rectilinear_args = tuple(_to_rectilinear(x) for x in args)
-    rectilinear_kwargs = {k: _to_rectilinear(v) for k, v in kwargs.items()}
+    backend = common_backend(_find_backends(chain(args, kwargs.values())))
+
+    rectilinear_args = tuple(_to_rectilinear(x, backend) for x in args)
+    rectilinear_kwargs = {k: _to_rectilinear(v, backend) for k, v in kwargs.items()}
     result = func(*rectilinear_args, **rectilinear_kwargs)
     # We want the result to be a layout (this will fail for functions returning non-array convertibles)
     out = ak.operations.ak_to_layout._impl(


Test patch:
diff --git a/tests/test_2305_nep_18_lazy_conversion.py b/tests/test_2305_nep_18_lazy_conversion.py
new file mode 100644
--- /dev/null
+++ b/tests/test_2305_nep_18_lazy_conversion.py
@@ -0,0 +1,24 @@
+# BSD 3-Clause License; see https://github.com/scikit-hep/awkward-1.0/blob/main/LICENSE
+
+import numpy as np
+import pytest
+
+import awkward as ak
+
+
+def test_binary():
+    ak_array = ak.Array(np.arange(10, dtype="<u4"))
+    np_array = np.arange(10, dtype=">u4")
+    assert np.array_equal(ak_array, np_array)
+
+
+def test_different_backend():
+    pytest.importorskip("jax")
+    ak.jax.register_and_check()
+
+    left = ak.Array([0, 1, 2], backend="jax")
+    right = ak.Array([0, 1, 2], backend="cpu")
+    with pytest.raises(
+        ValueError, match="cannot operate on arrays with incompatible backends"
+    ):
+        assert np.array_equal(left, right)