Repository: scikit-hep/awkward

Issue:
Array order dependent math failure between ListOffsetArray64 and ListOffsetArray64 containing UnionArray8_64
Forwarding https://github.com/CoffeaTeam/coffea/issues/367 here since it is mostly likely an issue in awkward given the trace (click link for details).

Repro is currently with NanoEvents to source data so you'll need the most recent coffea release to run this.
```python3
from coffea.nanoevents import NanoEventsFactory
fname = 'https://raw.githubusercontent.com/CoffeaTeam/coffea/master/tests/samples/nano_dy.root'
factory = NanoEventsFactory.from_file(fname)
events = factory.events()

gen_pt = awkward1.fill_none(events.Jet.matched_gen.pt, 0.)
jetPt = events.Jet.pt

print(-gen_pt + jetPt)  # produces correct answer
print(jetPt - gen_pt)  # fails with traceback as below
```
```
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/Users/lagray/miniconda3/envs/coffea-work/lib/python3.7/site-packages/numpy/lib/mixins.py", line 21, in func
    return ufunc(self, other)
  File "/Users/lagray/miniconda3/envs/coffea-work/lib/python3.7/site-packages/awkward1/highlevel.py", line 1369, in __array_ufunc__
    return awkward1._connect._numpy.array_ufunc(ufunc, method, inputs, kwargs)
  File "/Users/lagray/miniconda3/envs/coffea-work/lib/python3.7/site-packages/awkward1/_connect/_numpy.py", line 174, in array_ufunc
    inputs, getfunction, behavior, allow_records=False
  File "/Users/lagray/miniconda3/envs/coffea-work/lib/python3.7/site-packages/awkward1/_util.py", line 996, in broadcast_and_apply
    out = apply(broadcast_pack(inputs, isscalar), 0)
  File "/Users/lagray/miniconda3/envs/coffea-work/lib/python3.7/site-packages/awkward1/_util.py", line 759, in apply
    outcontent = apply(nextinputs, depth + 1)
  File "/Users/lagray/miniconda3/envs/coffea-work/lib/python3.7/site-packages/awkward1/_util.py", line 847, in apply
    outcontent = apply(nextinputs, depth + 1)
  File "/Users/lagray/miniconda3/envs/coffea-work/lib/python3.7/site-packages/awkward1/_util.py", line 589, in apply
    depth,
  File "/Users/lagray/miniconda3/envs/coffea-work/lib/python3.7/site-packages/awkward1/_util.py", line 655, in apply
    nextinputs.append(x[mask].project(combo[str(i)]))
ValueError: no field of name 1
```


Discussion:
Minimal reproducer:

```python
>>> content1 = ak.Array([0.0, 1.1, 2.2, 3.3, 4.4]).layout
>>> content2 = ak.Array([  0, 100, 200, 300, 400]).layout
>>> tags = ak.layout.Index8(np.array([0, 0, 0, 1, 1, 0, 0, 1, 1, 1], np.int8))
>>> index = ak.layout.Index64(np.array([0, 1, 2, 0, 1, 3, 4, 2, 3, 4], np.int64))
>>> unionarray = ak.Array(ak.layout.UnionArray8_64(tags, index, [content1, content2]))
>>> unionarray
<Array [0, 1.1, 2.2, 0, ... 4.4, 200, 300, 400] type='10 * union[float64, int64]'>
>>> unionarray + 10
<Array [10, 11.1, 12.2, 10, ... 210, 310, 410] type='10 * float64'>
>>> 10 + unionarray
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/home/jpivarski/miniconda3/lib/python3.8/site-packages/numpy/lib/mixins.py", line 31, in func
    return ufunc(other, self)
  File "/home/jpivarski/irishep/awkward-1.0/awkward1/highlevel.py", line 1369, in __array_ufunc__
    return awkward1._connect._numpy.array_ufunc(ufunc, method, inputs, kwargs)
  File "/home/jpivarski/irishep/awkward-1.0/awkward1/_connect/_numpy.py", line 173, in array_ufunc
    out = awkward1._util.broadcast_and_apply(
  File "/home/jpivarski/irishep/awkward-1.0/awkward1/_util.py", line 993, in broadcast_and_apply
    out = apply(broadcast_pack(inputs, isscalar), 0)
  File "/home/jpivarski/irishep/awkward-1.0/awkward1/_util.py", line 756, in apply
    outcontent = apply(nextinputs, depth + 1)
  File "/home/jpivarski/irishep/awkward-1.0/awkward1/_util.py", line 652, in apply
    nextinputs.append(x[mask].project(combo[str(i)]))
ValueError: no field of name 1
```

And you only get into this situation (of having a UnionArray) because of #459: floating point numbers with documentation are not being merged with floating point numbers without documentation. But anyway, I'm glad that the other bug exposed this one and will be fixing them both in one PR.

Resolving patch:
diff --git a/src/awkward1/_util.py b/src/awkward1/_util.py
--- a/src/awkward1/_util.py
+++ b/src/awkward1/_util.py
@@ -647,9 +647,11 @@ def apply(inputs, depth):
                 index[mask] = nplike.arange(nplike.count_nonzero(mask))
                 nextinputs = []
                 numoutputs = None
-                for i, x in enumerate(inputs):
+                i = 0
+                for x in inputs:
                     if isinstance(x, uniontypes):
                         nextinputs.append(x[mask].project(combo[str(i)]))
+                        i += 1
                     elif isinstance(x, awkward1.layout.Content):
                         nextinputs.append(x[mask])
                     else:
@@ -662,6 +664,7 @@ def apply(inputs, depth):
 
             tags = awkward1.layout.Index8(tags)
             index = awkward1.layout.Index64(index)
+
             return tuple(
                 awkward1.layout.UnionArray8_64(
                     tags, index, [x[i] for x in outcontents]


Test patch:
diff --git a/tests/test_0032-replace-dressedtype.py b/tests/test_0032-replace-dressedtype.py
--- a/tests/test_0032-replace-dressedtype.py
+++ b/tests/test_0032-replace-dressedtype.py
@@ -16,25 +16,25 @@
 def test_types_with_parameters():
     t = awkward1.types.UnknownType()
     assert t.parameters == {}
-    t.parameters = {"key": ["val", "ue"]}
-    assert t.parameters == {"key": ["val", "ue"]}
-    t = awkward1.types.UnknownType(parameters={"key": ["val", "ue"]})
-    assert t.parameters == {"key": ["val", "ue"]}
-
-    t = awkward1.types.PrimitiveType("int32", parameters={"key": ["val", "ue"]}); assert t.parameters == {"key": ["val", "ue"]}
-    t = awkward1.types.PrimitiveType("float64", parameters={"key": ["val", "ue"]}); assert t.parameters == {"key": ["val", "ue"]}
-    t = awkward1.types.ArrayType(awkward1.types.PrimitiveType("int32"), 100, parameters={"key": ["val", "ue"]}); assert t.parameters == {"key": ["val", "ue"]}
-    t = awkward1.types.ListType(awkward1.types.PrimitiveType("int32"), parameters={"key": ["val", "ue"]}); assert t.parameters == {"key": ["val", "ue"]}
-    t = awkward1.types.RegularType(awkward1.types.PrimitiveType("int32"), 5, parameters={"key": ["val", "ue"]}); assert t.parameters == {"key": ["val", "ue"]}
-    t = awkward1.types.OptionType(awkward1.types.PrimitiveType("int32"), parameters={"key": ["val", "ue"]}); assert t.parameters == {"key": ["val", "ue"]}
-    t = awkward1.types.UnionType((awkward1.types.PrimitiveType("int32"), awkward1.types.PrimitiveType("float64")), parameters={"key": ["val", "ue"]}); assert t.parameters == {"key": ["val", "ue"]}
-    t = awkward1.types.RecordType({"one": awkward1.types.PrimitiveType("int32"), "two": awkward1.types.PrimitiveType("float64")}, parameters={"key": ["val", "ue"]}); assert t.parameters == {"key": ["val", "ue"]}
-
-    t = awkward1.types.UnknownType(parameters={"key1": ["val", "ue"], "key2": u"one \u2192 two"})
-    assert t.parameters == {"key2": u"one \u2192 two", "key1": ["val", "ue"]}
-
-    assert t == awkward1.types.UnknownType(parameters={"key2": u"one \u2192 two", "key1": ["val", "ue"]})
-    assert t != awkward1.types.UnknownType(parameters={"key": ["val", "ue"]})
+    t.parameters = {"__array__": ["val", "ue"]}
+    assert t.parameters == {"__array__": ["val", "ue"]}
+    t = awkward1.types.UnknownType(parameters={"__array__": ["val", "ue"]})
+    assert t.parameters == {"__array__": ["val", "ue"]}
+
+    t = awkward1.types.PrimitiveType("int32", parameters={"__array__": ["val", "ue"]}); assert t.parameters == {"__array__": ["val", "ue"]}
+    t = awkward1.types.PrimitiveType("float64", parameters={"__array__": ["val", "ue"]}); assert t.parameters == {"__array__": ["val", "ue"]}
+    t = awkward1.types.ArrayType(awkward1.types.PrimitiveType("int32"), 100, parameters={"__array__": ["val", "ue"]}); assert t.parameters == {"__array__": ["val", "ue"]}
+    t = awkward1.types.ListType(awkward1.types.PrimitiveType("int32"), parameters={"__array__": ["val", "ue"]}); assert t.parameters == {"__array__": ["val", "ue"]}
+    t = awkward1.types.RegularType(awkward1.types.PrimitiveType("int32"), 5, parameters={"__array__": ["val", "ue"]}); assert t.parameters == {"__array__": ["val", "ue"]}
+    t = awkward1.types.OptionType(awkward1.types.PrimitiveType("int32"), parameters={"__array__": ["val", "ue"]}); assert t.parameters == {"__array__": ["val", "ue"]}
+    t = awkward1.types.UnionType((awkward1.types.PrimitiveType("int32"), awkward1.types.PrimitiveType("float64")), parameters={"__array__": ["val", "ue"]}); assert t.parameters == {"__array__": ["val", "ue"]}
+    t = awkward1.types.RecordType({"one": awkward1.types.PrimitiveType("int32"), "two": awkward1.types.PrimitiveType("float64")}, parameters={"__array__": ["val", "ue"]}); assert t.parameters == {"__array__": ["val", "ue"]}
+
+    t = awkward1.types.UnknownType(parameters={"key1": ["val", "ue"], "__record__": u"one \u2192 two"})
+    assert t.parameters == {"__record__": u"one \u2192 two", "key1": ["val", "ue"]}
+
+    assert t == awkward1.types.UnknownType(parameters={"__record__": u"one \u2192 two", "key1": ["val", "ue"]})
+    assert t != awkward1.types.UnknownType(parameters={"__array__": ["val", "ue"]})
 
 def test_dress():
     class Dummy(awkward1.highlevel.Array):
diff --git a/tests/test_0527-fix-unionarray-ufuncs-and-parameters-in-merging.py b/tests/test_0527-fix-unionarray-ufuncs-and-parameters-in-merging.py
new file mode 100644
--- /dev/null
+++ b/tests/test_0527-fix-unionarray-ufuncs-and-parameters-in-merging.py
@@ -0,0 +1,111 @@
+# BSD 3-Clause License; see https://github.com/scikit-hep/awkward-1.0/blob/master/LICENSE
+
+from __future__ import absolute_import
+
+import sys
+
+import pytest
+
+import numpy
+import awkward1
+
+
+# https://github.com/scikit-hep/awkward-1.0/issues/459#issuecomment-694941328
+#
+# So the rules would be,
+#    * if arrays have different `__array__` or `__record__` parameters, they are not equal;
+#    * if they otherwise have different parameters, the types can be equal, but merging
+#      (concatenation, option-simplify, or union-simplify) removes parameters other than
+#      `__array__` and `__record__`.
+
+
+def test_0459_types():
+    plain_plain = awkward1.Array([0.0, 1.1, 2.2, 3.3, 4.4])
+    array_plain = awkward1.with_parameter(plain_plain, "__array__", "zoinks")
+    plain_isdoc = awkward1.with_parameter(plain_plain, "__doc__", "This is a zoink.")
+    array_isdoc = awkward1.with_parameter(array_plain, "__doc__", "This is a zoink.")
+    assert awkward1.parameters(plain_plain) == {}
+    assert awkward1.parameters(array_plain) == {"__array__": "zoinks"}
+    assert awkward1.parameters(plain_isdoc) == {"__doc__": "This is a zoink."}
+    assert awkward1.parameters(array_isdoc) == {"__array__": "zoinks", "__doc__": "This is a zoink."}
+
+    assert awkward1.type(plain_plain) == awkward1.type(plain_plain)
+    assert awkward1.type(array_plain) == awkward1.type(array_plain)
+    assert awkward1.type(plain_isdoc) == awkward1.type(plain_isdoc)
+    assert awkward1.type(array_isdoc) == awkward1.type(array_isdoc)
+
+    assert awkward1.type(plain_plain) != awkward1.type(array_plain)
+    assert awkward1.type(array_plain) != awkward1.type(plain_plain)
+
+    assert awkward1.type(plain_plain) == awkward1.type(plain_isdoc)
+    assert awkward1.type(plain_isdoc) == awkward1.type(plain_plain)
+
+    assert awkward1.type(array_plain) == awkward1.type(array_isdoc)
+    assert awkward1.type(array_isdoc) == awkward1.type(array_plain)
+
+    assert awkward1.type(plain_isdoc) != awkward1.type(array_isdoc)
+    assert awkward1.type(array_isdoc) != awkward1.type(plain_isdoc)
+
+
+def test_0459():
+    plain_plain = awkward1.Array([0.0, 1.1, 2.2, 3.3, 4.4])
+    array_plain = awkward1.with_parameter(plain_plain, "__array__", "zoinks")
+    plain_isdoc = awkward1.with_parameter(plain_plain, "__doc__", "This is a zoink.")
+    array_isdoc = awkward1.with_parameter(array_plain, "__doc__", "This is a zoink.")
+    assert awkward1.parameters(plain_plain) == {}
+    assert awkward1.parameters(array_plain) == {"__array__": "zoinks"}
+    assert awkward1.parameters(plain_isdoc) == {"__doc__": "This is a zoink."}
+    assert awkward1.parameters(array_isdoc) == {"__array__": "zoinks", "__doc__": "This is a zoink."}
+
+    assert awkward1.parameters(awkward1.concatenate([plain_plain, plain_plain])) == {}
+    assert awkward1.parameters(awkward1.concatenate([array_plain, array_plain])) == {"__array__": "zoinks"}
+    assert awkward1.parameters(awkward1.concatenate([plain_isdoc, plain_isdoc])) == {"__doc__": "This is a zoink."}
+    assert awkward1.parameters(awkward1.concatenate([array_isdoc, array_isdoc])) == {"__array__": "zoinks", "__doc__": "This is a zoink."}
+
+    assert isinstance(awkward1.concatenate([plain_plain, plain_plain]).layout, awkward1.layout.NumpyArray)
+    assert isinstance(awkward1.concatenate([array_plain, array_plain]).layout, awkward1.layout.NumpyArray)
+    assert isinstance(awkward1.concatenate([plain_isdoc, plain_isdoc]).layout, awkward1.layout.NumpyArray)
+    assert isinstance(awkward1.concatenate([array_isdoc, array_isdoc]).layout, awkward1.layout.NumpyArray)
+
+    assert awkward1.parameters(awkward1.concatenate([plain_plain, array_plain])) == {}
+    assert awkward1.parameters(awkward1.concatenate([plain_isdoc, array_isdoc])) == {}
+    assert awkward1.parameters(awkward1.concatenate([array_plain, plain_plain])) == {}
+    assert awkward1.parameters(awkward1.concatenate([array_isdoc, plain_isdoc])) == {}
+
+    assert isinstance(awkward1.concatenate([plain_plain, array_plain]).layout, awkward1.layout.UnionArray8_64)
+    assert isinstance(awkward1.concatenate([plain_isdoc, array_isdoc]).layout, awkward1.layout.UnionArray8_64)
+    assert isinstance(awkward1.concatenate([array_plain, plain_plain]).layout, awkward1.layout.UnionArray8_64)
+    assert isinstance(awkward1.concatenate([array_isdoc, plain_isdoc]).layout, awkward1.layout.UnionArray8_64)
+
+    assert awkward1.parameters(awkward1.concatenate([plain_plain, plain_isdoc])) == {}
+    assert awkward1.parameters(awkward1.concatenate([array_plain, array_isdoc])) == {"__array__": "zoinks"}
+    assert awkward1.parameters(awkward1.concatenate([plain_isdoc, plain_plain])) == {}
+    assert awkward1.parameters(awkward1.concatenate([array_isdoc, array_plain])) == {"__array__": "zoinks"}
+
+    assert isinstance(awkward1.concatenate([plain_plain, plain_isdoc]).layout, awkward1.layout.NumpyArray)
+    assert isinstance(awkward1.concatenate([array_plain, array_isdoc]).layout, awkward1.layout.NumpyArray)
+    assert isinstance(awkward1.concatenate([plain_isdoc, plain_plain]).layout, awkward1.layout.NumpyArray)
+    assert isinstance(awkward1.concatenate([array_isdoc, array_plain]).layout, awkward1.layout.NumpyArray)
+
+
+def test_0522():
+    content1 = awkward1.Array([0.0, 1.1, 2.2, 3.3, 4.4]).layout
+    content2 = awkward1.Array([  0, 100, 200, 300, 400]).layout
+    tags = awkward1.layout.Index8(numpy.array([0, 0, 0, 1, 1, 0, 0, 1, 1, 1], numpy.int8))
+    index = awkward1.layout.Index64(numpy.array([0, 1, 2, 0, 1, 3, 4, 2, 3, 4], numpy.int64))
+    unionarray = awkward1.Array(awkward1.layout.UnionArray8_64(tags, index, [content1, content2]))
+    assert unionarray.tolist() == [0.0, 1.1, 2.2, 0, 100, 3.3, 4.4, 200, 300, 400]
+
+    assert (unionarray + 10).tolist() == [10.0, 11.1, 12.2, 10, 110, 13.3, 14.4, 210, 310, 410]
+    assert (10 + unionarray).tolist() == [10.0, 11.1, 12.2, 10, 110, 13.3, 14.4, 210, 310, 410]
+
+    assert (unionarray + range(0, 100, 10)).tolist() == [0.0, 11.1, 22.2, 30, 140, 53.3, 64.4, 270, 380, 490]
+    assert (range(0, 100, 10) + unionarray).tolist() == [0.0, 11.1, 22.2, 30, 140, 53.3, 64.4, 270, 380, 490]
+
+    assert (unionarray + numpy.arange(0, 100, 10)).tolist() == [0.0, 11.1, 22.2, 30, 140, 53.3, 64.4, 270, 380, 490]
+    assert (numpy.arange(0, 100, 10) + unionarray).tolist() == [0.0, 11.1, 22.2, 30, 140, 53.3, 64.4, 270, 380, 490]
+
+    assert (unionarray + awkward1.Array(numpy.arange(0, 100, 10))).tolist() == [0.0, 11.1, 22.2, 30, 140, 53.3, 64.4, 270, 380, 490]
+    assert (awkward1.Array(numpy.arange(0, 100, 10)) + unionarray).tolist() == [0.0, 11.1, 22.2, 30, 140, 53.3, 64.4, 270, 380, 490]
+
+    assert (unionarray + unionarray).tolist() == [0.0, 2.2, 4.4, 0, 200, 6.6, 8.8, 400, 600, 800]