Repository: scikit-hep/awkward

Issue:
Error saving with to_parquet - ValueError / Off-by-One
After updating to the latest `uproot` and `awkward`, I am having issues saving parquet files. I'm also having trouble figuring generating an example file to provide which reproduces the error. 

```
uproot  - 4.0.4
uproot3 - 3.14.3
awkward - 1.1.2
pyarrow - 3.0.0
```
This reliably reproduces the error
```
rootfiles = glob.glob("/path/to/rootfiles/*")
fields = \
[    
     'field1',
     'field2'
]
branches = [f"{filename}:events" for filename in rootfiles]
events = uproot.concatenate(branches[0:3], fields)
ak.to_parquet(events, "test.parquet")
```
The error 
```
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-23-c8be16528deb> in <module>
----> 1 ak.to_parquet(events, "test.parquet")

~/anaconda/playground/lib/python3.8/site-packages/awkward/operations/convert.py in to_parquet(array, where, explode_records, list_to32, string_to32, bytestring_to32, **options)
   2983     layout = to_layout(array, allow_record=False, allow_other=False)
   2984     iterator = batch_iterator(layout)
-> 2985     first = next(iterator)
   2986 
   2987     if "schema" not in options:

~/anaconda/playground/lib/python3.8/site-packages/awkward/operations/convert.py in batch_iterator(layout)
   2977                     )
   2978                 )
-> 2979             yield pyarrow.RecordBatch.from_arrays(
   2980                 pa_arrays, schema=pyarrow.schema(pa_fields)
   2981             )

~/anaconda/playground/lib/python3.8/site-packages/pyarrow/table.pxi in pyarrow.lib.RecordBatch.from_arrays()

ValueError: Arrays were not all the same length: 614 vs 615
```
However, checking the lengths seems to indicate that they are both 615
```
>>> for field in fields:
>>>     print(field, len(getattr(events, field)))
field1 615
field2 615
```
Examining the fields shows
```
>>> events.field1
<Array [454, 454, 454, 454, ... 174, 174, 174] type='615 * uint32'>
>>> events.field2
<Array [[0, 0, 0, 0, ... 296, 252, 0]] type='615 * var * float64'>
```

Now, unrelated to this bug report, but perhaps worth noting. I tried generating an example ROOT file to include, but ran into some issues. This is very likely some user error, though, but I have attached what I did manage to create.

[example_0.txt](https://github.com/scikit-hep/awkward-1.0/files/6013771/example_0.txt)
[example_1.txt](https://github.com/scikit-hep/awkward-1.0/files/6013772/example_1.txt)
[example_2.txt](https://github.com/scikit-hep/awkward-1.0/files/6013773/example_2.txt)

```
for i in range(3):
    file = uproot3.recreate(f"example_{i}.root")
    file["tree"] = uproot3.newtree({"branch1": np.int32, "branch2": np.float64}) # uint32 seems unsupported
    array1 = ak.to_numpy(events.field1)
    array2 = ak.to_numpy(events.field2)
    file["tree"].extend({"branch1": array1,
                                 "branch2": array2})
    file.close()

files = [f"example_{i}.root" for i in range(3)]
new_events = uproot.concatenate(files)

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-47-9aa3a03e189c> in <module>
      1 files = [f"example_{i}.root" for i in range(3)]
----> 2 new_events = uproot.concatenate(files)

~/anaconda/playground/lib/python3.8/site-packages/uproot/behaviors/TBranch.py in concatenate(files, expressions, cut, filter_name, filter_typename, filter_branch, aliases, language, decompression_executor, interpretation_executor, library, how, custom_classes, allow_missing, **options)
    371             with hasbranches:
    372                 try:
--> 373                     arrays = hasbranches.arrays(
    374                         expressions=expressions,
    375                         cut=cut,

~/anaconda/playground/lib/python3.8/site-packages/uproot/behaviors/TBranch.py in arrays(self, expressions, cut, filter_name, filter_typename, filter_branch, aliases, language, entry_start, entry_stop, decompression_executor, interpretation_executor, array_cache, library, how)
   1122                         ranges_or_baskets.append((branch, basket_num, range_or_basket))
   1123 
-> 1124         _ranges_or_baskets_to_arrays(
   1125             self,
   1126             ranges_or_baskets,

~/anaconda/playground/lib/python3.8/site-packages/uproot/behaviors/TBranch.py in _ranges_or_baskets_to_arrays(hasbranches, ranges_or_baskets, branchid_interpretation, entry_start, entry_stop, decompression_executor, interpretation_executor, library, arrays)
   3458 
   3459         elif isinstance(obj, tuple) and len(obj) == 3:
-> 3460             uproot.source.futures.delayed_raise(*obj)
   3461 
   3462         else:

~/anaconda/playground/lib/python3.8/site-packages/uproot/source/futures.py in delayed_raise(exception_class, exception_value, traceback)
     44         exec("raise exception_class, exception_value, traceback")
     45     else:
---> 46         raise exception_value.with_traceback(traceback)
     47 
     48 

~/anaconda/playground/lib/python3.8/site-packages/uproot/behaviors/TBranch.py in basket_to_array(basket)
   3415             )
   3416             if basket.num_entries != len(basket_arrays[basket.basket_num]):
-> 3417                 raise ValueError(
   3418                     """basket {0} in tree/branch {1} has the wrong number of entries """
   3419                     """(expected {2}, obtained {3}) when interpreted as {4}

ValueError: basket 0 in tree/branch /tree;1:branch2 has the wrong number of entries (expected 1, obtained 10) when interpreted as AsDtype('>f8')
    in file example_0.root
```

Thanks as always for all the hard work on awkward and uproot, and thanks further for any help you can provide!


Discussion:
For the last error, try the "low level interface" described on the Uproot 3 page: https://github.com/scikit-hep/uproot3#writing-ttrees

I was talking to someone on Gitter and they ran into a problem with `extend` that wasn't an issue with `newbasket`. Naturally, that's not a good situation, but this will get rewritten into Uproot 4 soon.

---------------

As for the first error, if you could make a small `events` and pickle it, then I'd have something to try. Slicing doesn't make the pickled array any smaller (that's another to-do item: #701), and slicing might affect the bug, anyway. Doing it with a small number of ROOT files and a small number of fields could make it manageable. If it's too big to post here or can't be posted publicly, we may be able to arrange something by email.
Well, this might actually be an issue with what gets returned from uproot. I have uploaded a pickle file. The actual field names are not `field1` and `field2`, but rather `file_number` and `num_electrons`.

This code confirms that pickling the object as returned from `uproot`, reloading it, and trying to save to parquet reproduces the error, so you should be able to duplicate the error with the attached pickle file.

```
with open('events.pickle', 'wb') as f:
    pickle.dump(events, f, pickle.HIGHEST_PROTOCOL)
with open('events.pickle', 'rb') as f:
    events_3 = pickle.load(f)
    
print(ak.all(events_3.file_number == events.file_number)) # Prints True
print(ak.all(events_3.num_electrons == events.num_electrons)) # Prints True

ak.to_parquet(events_3, "test_1.parquet") # Raises ValueError: Arrays were not all the same length: 614 vs 615
```

However, in attempting to make the file such that the fields were in fact called `field1` and `field2`, things seem to work.

```
# create a new object from the original as returned by uproot
events_1 = ak.zip({"field1": events.file_number, 
                   "field2": events.num_electrons})
print(ak.all(events_1.field1 == events.file_number)) # Prints True
print(ak.all(events_1.field2 == events.num_electrons)) # Prints True
ak.to_parquet(events_1, "test.parquet") # Succeeds

events_2 = ak.from_parquet("test.parquet") # Succeeds
print(ak.all(events_2.field1 == events.file_number)) # Prints True
print(ak.all(events_2.field2 == events.num_electrons)) # Prints True
```

So it looks less like an issue with the structure itself, and more to do with how it's getting returned from uproot? 

[events.txt](https://github.com/scikit-hep/awkward-1.0/files/6013919/events.txt)

As long as `events_3` is valid (check `ak.validity_error(events_3)`), it's not Uproot's fault.

Even though `events_1` and `events_2` have all the same elements as `events_3`, they can differ in `layout`, and it's probably a specific layout that isn't satisfying some rule for Arrow (Arrow is more strict than Awkward Array). That's why I needed the pickle file, to have exactly the layout that fails.

I'll take a look; thanks! (It might be Monday, though.)

Resolving patch:
diff --git a/src/awkward/_util.py b/src/awkward/_util.py
--- a/src/awkward/_util.py
+++ b/src/awkward/_util.py
@@ -914,9 +914,7 @@ def apply(inputs, depth, user):
         elif any(isinstance(x, recordtypes) for x in inputs):
             if not allow_records:
                 raise ValueError(
-                    "cannot broadcast: {0}".format(
-                        ", ".join(repr(type(x)) for x in inputs)
-                    )
+                    "cannot broadcast records in this type of operation"
                     + exception_suffix(__file__)
                 )
 
diff --git a/src/awkward/operations/convert.py b/src/awkward/operations/convert.py
--- a/src/awkward/operations/convert.py
+++ b/src/awkward/operations/convert.py
@@ -1950,49 +1950,6 @@ def _import_pyarrow(name):
         return pyarrow
 
 
-def _listarray_to_listoffsetarray(layout):
-    if isinstance(layout, ak.layout.ListArray32):
-        cls, index_cls = ak.layout.ListOffsetArray32, ak.layout.Index32
-    elif isinstance(layout, ak.layout.ListArrayU32):
-        cls, index_cls = ak.layout.ListOffsetArrayU32, ak.layout.IndexU32
-    elif isinstance(layout, ak.layout.ListArray64):
-        cls, index_cls = ak.layout.ListOffsetArray64, ak.layout.Index64
-    else:
-        cls = None
-
-    if cls is not None and layout.starts.ptr_lib == "cpu":
-        if numpy._module.array_equal(layout.starts[1:], layout.stops[:-1]):
-            offsets = index_cls(
-                numpy._module.append(numpy.asarray(layout.starts[1:]), layout.stops[-1])
-            )
-            return cls(offsets, layout.content, layout.identities, layout.parameters)
-
-    return layout
-
-
-def _regulararray_to_listoffsetarray(layout):
-    if isinstance(layout, ak.layout.RegularArray):
-        if layout.size == 0:
-            offsets = numpy.zeros(len(layout), dtype=np.int32)
-            cls, index_cls = ak.layout.ListOffsetArray32, ak.layout.Index32
-        else:
-            last = layout.size * (len(layout) + 1)
-            if last <= np.iinfo(np.int32).max:
-                offsets = numpy.arange(0, last, layout.size, dtype=np.int32)
-                cls, index_cls = ak.layout.ListOffsetArray32, ak.layout.Index32
-            elif last <= np.iinfo(np.uint32).max:
-                offsets = numpy.arange(0, last, layout.size, dtype=np.uint32)
-                cls, index_cls = ak.layout.ListOffsetArrayU32, ak.layout.IndexU32
-            else:
-                offsets = numpy.arange(0, last, layout.size, dtype=np.int64)
-                cls, index_cls = ak.layout.ListOffsetArray64, ak.layout.Index64
-            return cls(
-                index_cls(offsets), layout.content, layout.identities, layout.parameters
-            )
-
-    return layout
-
-
 def to_arrow(array, list_to32=False, string_to32=True, bytestring_to32=True):
     """
     Args:
@@ -2022,8 +1979,6 @@ def to_arrow(array, list_to32=False, string_to32=True, bytestring_to32=True):
     layout = to_layout(array, allow_record=False, allow_other=False)
 
     def recurse(layout, mask, is_option):
-        layout = _regulararray_to_listoffsetarray(_listarray_to_listoffsetarray(layout))
-
         if isinstance(layout, ak.layout.NumpyArray):
             numpy_arr = numpy.asarray(layout)
             length = len(numpy_arr)


Test patch:
diff --git a/tests/test_0224-arrow-to-awkward.py b/tests/test_0224-arrow-to-awkward.py
--- a/tests/test_0224-arrow-to-awkward.py
+++ b/tests/test_0224-arrow-to-awkward.py
@@ -110,7 +110,7 @@ def test_toarrow_ListArray_RegularArray():
         [[[3.3, 4.4], [5.5]], [[6.6, 7.7, 8.8, 9.9], []]],
     ]
 
-    assert isinstance(ak.to_arrow(regulararray), (pyarrow.ListArray))
+    assert isinstance(ak.to_arrow(regulararray), (pyarrow.LargeListArray))
     assert ak.to_arrow(regulararray).to_pylist() == [
         [[0.0, 1.1, 2.2], []],
         [[3.3, 4.4], [5.5]],