Repository: scikit-hep/awkward

Issue:
Make Form::getitem_field correctly mirror Content::getitem_field
I closed PR #390 as-is because it solves @nsmith-'s problem, but `Form::getitem_field` is defined in a non-intuitive way.

https://github.com/scikit-hep/awkward-1.0/blob/6a3e6bed7fb5312d3190f2299abca71299b1a86c/include/awkward/Content.h#L226-L234

This issue is to clean that up and make sure that @nsmith-'s tests still work. Maybe we should add `Form::getitem_fields` as well, since it's nearly the same thing and would be expected to have the same materializing/non-materializing behavior.

This is good first issue for someone interested in how these nodes fit together and is willing to delve into C++.


Discussion:

Resolving patch:
diff --git a/src/awkward1/_connect/_numba/arrayview.py b/src/awkward1/_connect/_numba/arrayview.py
--- a/src/awkward1/_connect/_numba/arrayview.py
+++ b/src/awkward1/_connect/_numba/arrayview.py
@@ -321,6 +321,11 @@ def fromarray(self, array):
             allow_other=False,
             numpytype=(np.number, np.bool_, np.bool),
         )
+        while (
+            isinstance(layout, awkward1.layout.VirtualArray)
+            and isinstance(layout.generator, awkward1.layout.SliceGenerator)
+        ):
+            layout = layout.array
         layout = awkward1.operations.convert.regularize_numpyarray(
             layout, allow_empty=False, highlevel=False
         )
diff --git a/src/awkward1/_util.py b/src/awkward1/_util.py
--- a/src/awkward1/_util.py
+++ b/src/awkward1/_util.py
@@ -550,16 +550,13 @@ def apply(inputs, depth):
         nplike = awkward1.nplike.of(*inputs)
 
         # handle implicit right-broadcasting (i.e. NumPy-like)
-        if any(isinstance(x, listtypes) for x in inputs) and not any(
-            isinstance(x, (awkward1.layout.Content, awkward1.layout.Record))
-            and x.has_virtual_form
-            for x in inputs
-        ):
+        if any(isinstance(x, listtypes) for x in inputs):
             maxdepth = max(
                 x.purelist_depth
                 for x in inputs
                 if isinstance(x, awkward1.layout.Content)
             )
+
             if maxdepth > 0 and all(
                 x.purelist_isregular
                 for x in inputs


Test patch:
diff --git a/tests/test_0230-virtualarray-segfault.py b/tests/test_0230-virtualarray-segfault.py
--- a/tests/test_0230-virtualarray-segfault.py
+++ b/tests/test_0230-virtualarray-segfault.py
@@ -11,7 +11,7 @@
 import awkward1._io
 
 def test_0230():
-    rec = awkward1.zip({"x": awkward1.virtual(lambda: awkward1.Array([1, 2, 3, 4]), length=4),},
+    rec = awkward1.zip({"x": awkward1.virtual(lambda: awkward1.Array([1, 2, 3, 4]), length=4)},
                        depth_limit=1)
     assert awkward1.to_list(rec.x[1:]) == [2, 3, 4]
     assert awkward1.to_list(rec.x[1:]*2) == [4, 6, 8]
diff --git a/tests/test_0390-virtual-forms.py b/tests/test_0390-virtual-forms.py
--- a/tests/test_0390-virtual-forms.py
+++ b/tests/test_0390-virtual-forms.py
@@ -3,6 +3,8 @@
 from __future__ import absolute_import
 from collections import defaultdict
 
+import json
+
 import pytest
 import awkward1
 
@@ -66,11 +68,10 @@ def gen(x, name):
     x2 = awkward1.Array([1, 2, 3, 4, 5])
     x = awkward1.zip({"x1": x1, "x2": x2}, with_name="xthing")
     xv = awkward1.virtual(lambda: gen(x, "x"), length=len(x), form=x.layout.form)
-    y = x1 * 10.
+    y = x1 * 10.0
     yv = awkward1.virtual(lambda: gen(y, "y"), length=len(y), form=y.layout.form)
     array = awkward1.zip({"x": xv, "y": yv}, with_name="Point", depth_limit=1)
-    virtual = awkward1.virtual(lambda: gen(array, "array"), length=len(array), form=array.layout.form)
-
+    virtual = awkward1.virtual(lambda: gen(array, "array"), length=len(array), form=awkward1.forms.Form.fromjson(json.dumps({"class": "RecordArray", "contents": {"x": {"class": "VirtualArray", "form": json.loads(str(x.layout.form)), "has_length": True}, "y": {"class": "VirtualArray", "form": json.loads(str(y.layout.form)), "has_length": True}}, "parameters": {"__record__": "Point"}})))
 
     @numba.njit
     def dostuff(array):
@@ -80,7 +81,7 @@ def dostuff(array):
         return x
 
     assert dostuff(virtual.x.x1) == 15
-    assert dict(materialize_count) == {"x": 3, "array": 3}
+    assert dict(materialize_count) == {"x": 1, "array": 1}
     materialize_count.clear()
 
     @numba.njit
diff --git a/tests/test_0402-fix-form-getitem-field.py b/tests/test_0402-fix-form-getitem-field.py
new file mode 100644
--- /dev/null
+++ b/tests/test_0402-fix-form-getitem-field.py
@@ -0,0 +1,74 @@
+# BSD 3-Clause License; see https://github.com/scikit-hep/awkward-1.0/blob/master/LICENSE
+
+from __future__ import absolute_import
+
+import sys
+import json
+
+import pytest
+
+import numpy
+import awkward1
+
+
+def test():
+    example = awkward1.Array([[{"x": 1, "y": []}, {"x": 2, "y": [1, 1]}], []])
+    cache = {}
+    virtualarray = awkward1.virtual(lambda: example, form=example.layout.form, length=len(example))
+    assert len(cache) == 0
+
+    tmp1 = virtualarray["x"]
+    assert json.loads(str(tmp1.layout.form)) == {
+        "class": "VirtualArray",
+        "form": {
+            "class": "ListOffsetArray64",
+            "offsets": "i64",
+            "content": "int64"
+        },
+        "has_length": True
+    }
+    assert len(cache) == 0
+
+    tmp2 = virtualarray["y"]
+    assert json.loads(str(tmp2.layout.form)) == {
+        "class": "VirtualArray",
+        "form": {
+            "class": "ListOffsetArray64",
+            "offsets": "i64",
+            "content": {
+                "class": "ListOffsetArray64",
+                "offsets": "i64",
+                "content": "int64"
+            }
+        },
+        "has_length": True
+    }
+    assert len(cache) == 0
+
+    assert tmp1.tolist() == [[1, 2], []]
+    assert tmp2.tolist() == [[[], [1, 1]], []]
+
+def test_no_break_regular_broadcasting():
+    # because we redefined Form::has_virtual_form to agree with an interpretation of its name in English
+
+    assert (numpy.array([[1, 2, 3], [4, 5, 6]]) + numpy.array([[10], [20]])).tolist() == [[11, 12, 13], [24, 25, 26]]
+    assert (awkward1.Array(numpy.array([[1, 2, 3], [4, 5, 6]])) + awkward1.Array(numpy.array([[10], [20]]))).tolist() == [[11, 12, 13], [24, 25, 26]]
+    with pytest.raises(ValueError):
+        awkward1.Array([[1, 2, 3], [4, 5, 6]]) + awkward1.Array([[10], [20]])
+    left, right = awkward1.Array(numpy.array([[1, 2, 3], [4, 5, 6]])), awkward1.Array(numpy.array([[10], [20]]))
+    assert (awkward1.virtual(lambda: left, form=left.layout.form, length=len(left)) + awkward1.virtual(lambda: right, form=right.layout.form, length=len(right))).tolist() == [[11, 12, 13], [24, 25, 26]]
+    with pytest.raises(ValueError):
+        left, right = awkward1.Array([[1, 2, 3], [4, 5, 6]]), awkward1.Array([[10], [20]])
+        awkward1.virtual(lambda: left, form=left.layout.form, length=len(left)) + awkward1.virtual(lambda: right, form=right.layout.form, length=len(right))
+
+    assert (numpy.array([[1, 2, 3], [4, 5, 6]]) + numpy.array([10])).tolist() == [[11, 12, 13], [14, 15, 16]]
+    assert (awkward1.Array(numpy.array([[1, 2, 3], [4, 5, 6]])) + awkward1.Array(numpy.array([10]))).tolist() == [[11, 12, 13], [14, 15, 16]]
+    assert (awkward1.Array([[1, 2, 3], [4, 5, 6]]) + awkward1.Array([10])).tolist() == [[11, 12, 13], [14, 15, 16]]
+    left, right = awkward1.Array(numpy.array([[1, 2, 3], [4, 5, 6]])), awkward1.Array(numpy.array([10]))
+    assert (awkward1.virtual(lambda: left, form=left.layout.form, length=len(left)) + awkward1.virtual(lambda: right, form=right.layout.form, length=len(right))).tolist() == [[11, 12, 13], [14, 15, 16]]
+    left, right = awkward1.Array([[1, 2, 3], [4, 5, 6]]), awkward1.Array([10])
+    assert (awkward1.virtual(lambda: left, form=left.layout.form, length=len(left)) + awkward1.virtual(lambda: right, form=right.layout.form, length=len(right))).tolist() == [[11, 12, 13], [14, 15, 16]]
+
+    assert (awkward1.Array([[1, 2, 3], [4, 5, 6]]) + awkward1.Array([10, 20])).tolist() == [[11, 12, 13], [24, 25, 26]]
+    left, right = awkward1.Array([[1, 2, 3], [4, 5, 6]]), awkward1.Array([10, 20])
+    assert (awkward1.virtual(lambda: left, form=left.layout.form, length=len(left)) + awkward1.virtual(lambda: right, form=right.layout.form, length=len(right))).tolist() == [[11, 12, 13], [24, 25, 26]]