Repository: scikit-hep/awkward

Issue:
PR #2477 broke an Uproot test
### Version of Awkward Array

anything after 13aea6de849a794ccb35b4743edc1602dcd97f7a (#2477)

### Description and code to reproduce

Suppose that a `ListOffsetArray` contains unreachable elements, like

```python
import awkward as ak
import numpy as np

a = ak.Array(
    ak.contents.ListOffsetArray(
        ak.index.Index64(np.array([1, 2, 3], np.int64)),
        ak.contents.NumpyArray(
            np.array([1, 1, 2, 1, 2, 3, 1, 2, 3, 4, 1, 2, 3, 4, 5], np.int64)
        ),
    )
)

assert a.to_list() == [[1], [2]]
```

(We get exactly this structure in an [Uproot test](https://github.com/scikit-hep/uproot5/blob/366e3dcdca8fffb61f9c770897d71197809a96f4/tests/test_0662-rntuple-stl-containers.py#L42).)

When a ufunc is applied to it, such as `np.equal` in

```python
assert ak.all(a == [[1], [2]])
```

broadcasting fails to trim or project while descending:

```
Traceback (most recent call last):
  File "/home/jpivarski/irishep/awkward/src/awkward/highlevel.py", line 1362, in __array_ufunc__
    return ak._connect.numpy.array_ufunc(ufunc, method, inputs, kwargs)
  File "/home/jpivarski/irishep/awkward/src/awkward/_connect/numpy.py", line 321, in array_ufunc
    out = ak._broadcasting.broadcast_and_apply(
  File "/home/jpivarski/irishep/awkward/src/awkward/_broadcasting.py", line 1098, in broadcast_and_apply
    out = apply_step(
  File "/home/jpivarski/irishep/awkward/src/awkward/_broadcasting.py", line 1077, in apply_step
    return continuation()
  File "/home/jpivarski/irishep/awkward/src/awkward/_broadcasting.py", line 1050, in continuation
    return broadcast_any_list()
  File "/home/jpivarski/irishep/awkward/src/awkward/_broadcasting.py", line 580, in broadcast_any_list
    outcontent = apply_step(
  File "/home/jpivarski/irishep/awkward/src/awkward/_broadcasting.py", line 1077, in apply_step
    return continuation()
  File "/home/jpivarski/irishep/awkward/src/awkward/_broadcasting.py", line 1050, in continuation
    return broadcast_any_list()
  File "/home/jpivarski/irishep/awkward/src/awkward/_broadcasting.py", line 771, in broadcast_any_list
    outcontent = apply_step(
  File "/home/jpivarski/irishep/awkward/src/awkward/_broadcasting.py", line 407, in apply_step
    checklength(contents, options)
  File "/home/jpivarski/irishep/awkward/src/awkward/_broadcasting.py", line 116, in checklength
    raise ValueError(
ValueError: cannot broadcast NumpyArray of length 14 with NumpyArray of length 2in equal

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/home/jpivarski/irishep/awkward/testy.py", line 15, in <module>
    assert ak.all(a == [[1], [2]])
  File "/home/jpivarski/mambaforge/lib/python3.9/site-packages/numpy/lib/mixins.py", line 21, in func
    return ufunc(self, other)
  File "/home/jpivarski/irishep/awkward/src/awkward/highlevel.py", line 1362, in __array_ufunc__
    return ak._connect.numpy.array_ufunc(ufunc, method, inputs, kwargs)
  File "/home/jpivarski/irishep/awkward/src/awkward/_errors.py", line 56, in __exit__
    self.handle_exception(exception_type, exception_value)
  File "/home/jpivarski/irishep/awkward/src/awkward/_errors.py", line 71, in handle_exception
    raise self.decorate_exception(cls, exception)
ValueError: cannot broadcast NumpyArray of length 14 with NumpyArray of length 2in equal

This error occurred while calling

    numpy.equal.__call__(
        <Array [[1], [2]] type='2 * var * int64'>
        [[1], [2]]
    )
```

I did a git bisect by hand and narrowed it down to PR #2477. ("By hand" because there's an awkward-cpp change that needs a recompilation, and I didn't want to recompile in every test.) It's fine before that change and fails with the above error afterward.

Fortunately, this didn't go into any release, and it will need to be fixed before the next release. Also, we apparently needed a test like this—perhaps too many of our tests check equality with `to_list`.


Discussion:

Resolving patch:
diff --git a/src/awkward/contents/listoffsetarray.py b/src/awkward/contents/listoffsetarray.py
--- a/src/awkward/contents/listoffsetarray.py
+++ b/src/awkward/contents/listoffsetarray.py
@@ -383,20 +383,22 @@ def _broadcast_tooffsets64(self, offsets: Index) -> ListOffsetArray:
             )
 
         # Check whether we need to slice the content, shift our offsets
-        start = self._offsets[0]
+        this_start = self._offsets[0]
         this_zero_offsets = self._offsets.data
-        if index_nplike.known_data and start == 0:
+        if index_nplike.known_data and this_start == 0:
             next_content = self._content
         else:
-            this_zero_offsets = this_zero_offsets - start
-            next_content = self._content[start:]
+            this_zero_offsets = this_zero_offsets - this_start
+            next_content = self._content[this_start:]
 
         if index_nplike.known_data and not index_nplike.array_equal(
             this_zero_offsets, offsets
         ):
             raise ValueError("cannot broadcast nested list")
 
-        return ListOffsetArray(offsets, next_content, parameters=self._parameters)
+        return ListOffsetArray(
+            offsets, next_content[: offsets[-1]], parameters=self._parameters
+        )
 
     def _getitem_next_jagged(
         self, slicestarts: Index, slicestops: Index, slicecontent: Content, tail


Test patch:
diff --git a/tests/test_2487_broadcast_list_offsets.py b/tests/test_2487_broadcast_list_offsets.py
new file mode 100644
--- /dev/null
+++ b/tests/test_2487_broadcast_list_offsets.py
@@ -0,0 +1,21 @@
+# BSD 3-Clause License; see https://github.com/scikit-hep/awkward-1.0/blob/main/LICENSE
+
+import numpy as np
+import pytest  # noqa: F401
+
+import awkward as ak
+
+
+def test_reducer():
+    # broadcast_tooffsets64 was changed, and stopped trimming the content
+    # this test ensures that things still work!
+    a = ak.Array(
+        ak.contents.ListOffsetArray(
+            ak.index.Index64(np.array([1, 2, 3], dtype=np.int64)),
+            ak.contents.NumpyArray(
+                np.array([1, 1, 2, 1, 2, 3, 1, 2, 3, 4, 1, 2, 3, 4, 5], dtype=np.int64)
+            ),
+        )
+    )
+    # Perform broadcast, check result works!
+    assert ak.all(a == [[1], [2]])