Repository: vaexio/vaex

Issue:
Failing to open arrow stream with categoricals
Vaex fails to open arrow streams that contain categorical columns. It would be great if this was working as categoricals have given a great performance in many of my applications.

```import pandas as pd
import pyarrow as pa
import numpy as np
import vaex
df = pd.DataFrame(
    {
        'col1': pd.Categorical.from_codes(np.full(1, 1), categories=['ABC', 'DEF'])
    }
)
table = pa.Table.from_pandas(df)

with pa.OSFile('test2.arrow', 'wb') as sink:
    with pa.RecordBatchStreamWriter(sink, table.schema) as writer:
        writer.write_table(table)

with pa.OSFile('test2.arrow', 'rb') as source:
    df = pa.ipc.open_stream(source).read_pandas()

df = vaex.open('test2.arrow')
```

Output:
```
ERROR:MainThread:vaex:error opening 'test2.arrow'
---------------------------------------------------------------------------
NotImplementedError                       Traceback (most recent call last)
 in 
     17     df = pa.ipc.open_stream(source).read_pandas()
     18 
---> 19 df = vaex.open('test2.arrow')

Z:\Systemdateien\Miniconda3\envs\finance\lib\site-packages\vaex\__init__.py in open(path, convert, shuffle, copy_index, *args, **kwargs)
    189                         ds = from_csv(path, copy_index=copy_index, **kwargs)
    190                     else:
--> 191                         ds = vaex.file.open(path, *args, **kwargs)
    192                     if convert and ds:
    193                         ds.export_hdf5(filename_hdf5, shuffle=shuffle)

Z:\Systemdateien\Miniconda3\envs\finance\lib\site-packages\vaex\file\__init__.py in open(path, *args, **kwargs)
     28     for opener in opener_classes:
     29         if opener.can_open(path, *args, **kwargs):
---> 30             return opener.open(path, *args, **kwargs)
     31     if hdf5:
     32         openers.extend(hdf5.dataset.dataset_type_map.items())

Z:\Systemdateien\Miniconda3\envs\finance\lib\site-packages\vaex_arrow\opener.py in open(path, *args, **kwargs)
      9     def open(path, *args, **kwargs):
     10         from .dataset import DatasetArrow
---> 11         return DatasetArrow(path, *args, **kwargs)
     12 
     13 class ParquetOpener:

Z:\Systemdateien\Miniconda3\envs\finance\lib\site-packages\vaex_arrow\dataset.py in __init__(self, filename, table, write)
     18         self._write = write
     19         if table is None:
---> 20             self._load()
     21         else:
     22             self._load_table(table)

Z:\Systemdateien\Miniconda3\envs\finance\lib\site-packages\vaex_arrow\dataset.py in _load(self)
     26         reader = pa.ipc.open_stream(source)
     27         table = pa.Table.from_batches([b for b in reader])
---> 28         self._load_table(table)
     29 
     30     def _load_table(self, table):

Z:\Systemdateien\Miniconda3\envs\finance\lib\site-packages\vaex_arrow\dataset.py in _load_table(self, table)
     35             # TODO: keep the arrow columns, and support and test chunks
     36             arrow_array = col.data.chunks[0]
---> 37             column = column_from_arrow_array(arrow_array)
     38 
     39             self.columns[name] = column

Z:\Systemdateien\Miniconda3\envs\finance\lib\site-packages\vaex_arrow\convert.py in column_from_arrow_array(arrow_array)
     28     buffers = arrow_array.buffers()
     29     if len(buffers) == 2:
---> 30         return numpy_array_from_arrow_array(arrow_array)
     31     elif len(buffers) == 3 and  isinstance(arrow_array.type, type(pyarrow.string())):
     32         bitmap_buffer, offsets, string_bytes = arrow_array.buffers()

Z:\Systemdateien\Miniconda3\envs\finance\lib\site-packages\vaex_arrow\convert.py in numpy_array_from_arrow_array(arrow_array)
     62         array = np.frombuffer(buffers[-1], dtype, len(arrow_array))# TODO: deal with offset ? [arrow_array.offset:arrow_array.offset + len(arrow_array)]
     63     else:
---> 64         dtype = arrow_array.type.to_pandas_dtype()
     65     if np.bool_ == dtype:
     66         # TODO: this will also be a copy, we probably want to support bitmasks as well

Z:\Systemdateien\Miniconda3\envs\finance\lib\site-packages\pyarrow\types.pxi in pyarrow.lib.DataType.to_pandas_dtype()

NotImplementedError: dictionary
```


Discussion:
I have to see how this maps to vaex, but yes, this is something we should support. Thanks again for the great issue! 
Not sure if it helps as the problem in the following references was parquet, but maybe it can provide some inspiration:
https://issues.apache.org/jira/browse/ARROW-3246
Associated PR: apache/arrow#5077


Resolving patch:
diff --git a/packages/vaex-arrow/vaex_arrow/dataset.py b/packages/vaex-arrow/vaex_arrow/dataset.py
--- a/packages/vaex-arrow/vaex_arrow/dataset.py
+++ b/packages/vaex-arrow/vaex_arrow/dataset.py
@@ -28,14 +28,18 @@ def _load(self):
         self._load_table(table)
     
     def _load_table(self, table):
-        self._length_unfiltered =  self._length_original = table.num_rows
-        self._index_end =  self._length_original = table.num_rows
+        self._length_unfiltered = self._length_original = table.num_rows
+        self._index_end = self._length_original = table.num_rows
         for col in table.columns:
             name = col.name
             # TODO: keep the arrow columns, and support and test chunks
             arrow_array = col.data.chunks[0]
-            column = column_from_arrow_array(arrow_array)
-
+            if isinstance(arrow_array.type, pa.DictionaryType):
+                column = column_from_arrow_array(arrow_array.indices)
+                labels = column_from_arrow_array(arrow_array.dictionary).tolist()
+                self._categories[name] = dict(labels=labels, N=len(labels))
+            else:
+                column = column_from_arrow_array(arrow_array)
             self.columns[name] = column
             self.column_names.append(name)
             self._save_assign_expression(name, vaex.expression.Expression(self, name))


Test patch: