Repository: vaexio/vaex

Issue:
Failing to open arrow file format & misleading error message
Vaex fails to open arrow file format. It confusingly expects a file written not in the arrow file-format but in arrow stream-format.

If this is a non-negotiable vaex constraint, a less misleading error message might help very confused novices...

```

import pandas as pd
import pyarrow as pa
import numpy as np
import vaex
df = pd.DataFrame(
    {
        'col1': range(5)
    }
)
table = pa.Table.from_pandas(df)

with pa.OSFile('test2.arrow', 'wb') as sink:
    with pa.RecordBatchFileWriter(sink, table.schema) as writer:
        writer.write_table(table)

with pa.OSFile('test2.arrow', 'rb') as source:
    df = pa.ipc.open_file(source).read_pandas()
df = vaex.open('test2.arrow')
```

Error messages:
```
ERROR:MainThread:vaex:error opening 'test2.arrow'
---------------------------------------------------------------------------
ArrowInvalid                              Traceback (most recent call last)
 in 
     16 with pa.OSFile('test2.arrow', 'rb') as source:
     17     df = pa.ipc.open_file(source).read_pandas()
---> 18 df = vaex.open('test2.arrow')

Z:\Systemdateien\Miniconda3\envs\finance\lib\site-packages\vaex\__init__.py in open(path, convert, shuffle, copy_index, *args, **kwargs)
    189                         ds = from_csv(path, copy_index=copy_index, **kwargs)
    190                     else:
--> 191                         ds = vaex.file.open(path, *args, **kwargs)
    192                     if convert and ds:
    193                         ds.export_hdf5(filename_hdf5, shuffle=shuffle)

Z:\Systemdateien\Miniconda3\envs\finance\lib\site-packages\vaex\file\__init__.py in open(path, *args, **kwargs)
     28     for opener in opener_classes:
     29         if opener.can_open(path, *args, **kwargs):
---> 30             return opener.open(path, *args, **kwargs)
     31     if hdf5:
     32         openers.extend(hdf5.dataset.dataset_type_map.items())

Z:\Systemdateien\Miniconda3\envs\finance\lib\site-packages\vaex_arrow\opener.py in open(path, *args, **kwargs)
      9     def open(path, *args, **kwargs):
     10         from .dataset import DatasetArrow
---> 11         return DatasetArrow(path, *args, **kwargs)
     12 
     13 class ParquetOpener:

Z:\Systemdateien\Miniconda3\envs\finance\lib\site-packages\vaex_arrow\dataset.py in __init__(self, filename, table, write)
     18         self._write = write
     19         if table is None:
---> 20             self._load()
     21         else:
     22             self._load_table(table)

Z:\Systemdateien\Miniconda3\envs\finance\lib\site-packages\vaex_arrow\dataset.py in _load(self)
     24     def _load(self):
     25         source = pa.memory_map(self.path)
---> 26         reader = pa.ipc.open_stream(source)
     27         table = pa.Table.from_batches([b for b in reader])
     28         self._load_table(table)

Z:\Systemdateien\Miniconda3\envs\finance\lib\site-packages\pyarrow\ipc.py in open_stream(source)
    123     reader : RecordBatchStreamReader
    124     """
--> 125     return RecordBatchStreamReader(source)
    126 
    127 

Z:\Systemdateien\Miniconda3\envs\finance\lib\site-packages\pyarrow\ipc.py in __init__(self, source)
     58     """
     59     def __init__(self, source):
---> 60         self._open(source)
     61 
     62 

Z:\Systemdateien\Miniconda3\envs\finance\lib\site-packages\pyarrow\ipc.pxi in pyarrow.lib._RecordBatchStreamReader._open()

Z:\Systemdateien\Miniconda3\envs\finance\lib\site-packages\pyarrow\error.pxi in pyarrow.lib.check_status()

ArrowInvalid: Expected to read 1330795073 metadata bytes, but only read 1474
```


Discussion:
Hi,

thanks for the clear report. I think we should support this.

cheesr,

Maarten

Resolving patch:
diff --git a/packages/vaex-arrow/vaex_arrow/dataset.py b/packages/vaex-arrow/vaex_arrow/dataset.py
--- a/packages/vaex-arrow/vaex_arrow/dataset.py
+++ b/packages/vaex-arrow/vaex_arrow/dataset.py
@@ -23,8 +23,18 @@ def __init__(self, filename=None, table=None, write=False):
 
     def _load(self):
         source = pa.memory_map(self.path)
-        reader = pa.ipc.open_stream(source)
-        table = pa.Table.from_batches([b for b in reader])
+        try:
+            # first we try if it opens as stream
+            reader = pa.ipc.open_stream(source)
+        except pa.lib.ArrowInvalid:
+            # if not, we open as file
+            reader = pa.ipc.open_file(source)
+            # for some reason this reader is not iterable
+            batches = [reader.get_batch(i) for i in range(reader.num_record_batches)]
+        else:
+            # if a stream, we're good
+            batches = reader  # this reader is iterable
+        table = pa.Table.from_batches(batches)
         self._load_table(table)
     
     def _load_table(self, table):


Test patch:
diff --git a/tests/arrow/stream_file_test.py b/tests/arrow/stream_file_test.py
new file mode 100644
--- /dev/null
+++ b/tests/arrow/stream_file_test.py
@@ -0,0 +1,23 @@
+import pandas as pd
+import pyarrow as pa
+import numpy as np
+import vaex
+
+df = pd.DataFrame({'col1': range(5)})
+table = pa.Table.from_pandas(df)
+
+def test_arrow_write_table(tmpdir):
+    path = str(tmpdir.join('test.arrow'))
+    with pa.OSFile(path, 'wb') as sink:
+        with pa.RecordBatchFileWriter(sink, table.schema) as writer:
+            writer.write_table(table)
+
+    df = vaex.open(path)
+
+def test_arrow_write_stream(tmpdir):
+    path = str(tmpdir.join('test.arrow'))
+    with pa.OSFile(path, 'wb') as sink:
+        with pa.RecordBatchStreamWriter(sink, table.schema) as writer:
+            writer.write_table(table)
+
+    df = vaex.open(path)