Repository: vaexio/vaex

Issue:
[BUG-REPORT] Can't Convert PyArrow Float Col to Str
**Description**
I encountered this strange edge case when trying to cast a PyArrow float column (well, technically a vaex column from an original PyArrow table) to string. For some reason, the object returned by `vaex.arrow.convert.column_from_arrow_array(x)` is of type `np.ndarray` and therefore throwing an error when trying to access the `string_sequence` attribute during the actual dtype conversion process. I imagine that something strange is going on under the hood in how the type conversion is being handled. The full traceback can be found below. I saw the error when I used pyarrow==1.0.1 as well as pyarrow==2.0.0.

I encountered this bug while employing a workaround for [#1001](https://github.com/vaexio/vaex/issues/1001#issuecomment-704621902). So while I really shouldn't need to use this to/from arrow table workaround long-term, I definitely think that the behavior I'm seeing below is unexpected.

**Software information**
 - Vaex version (`import vaex; vaex.__version__)`:
'vaex': '4.0.0a6',
 'vaex-core': '4.0.0a11',
 'vaex-viz': '0.5.0.dev1',
 'vaex-hdf5': '0.7.0a5',
 'vaex-server': '0.4.0a1',
 'vaex-astro': '0.8.0.dev1',
 'vaex-jupyter': '0.6.0.dev1',
 'vaex-ml': '0.11.0a4'
 - Vaex was installed via: pip / conda-forge / from source: pip install vaex==4.0.0a6
 - OS: Windows 10

**Additional information**
```
import numpy as np
import vaex

# Works:
dt1 = vaex.from_arrays(
    val=np.array([np.nan, np.nan, np.nan], dtype='float64')
)
print(dt1.dtypes)
dt1['val'] = dt1['val'].astype('string')
print(dt1.dtypes)

# Breaks
dt2 = vaex.from_arrays(
    val=np.array([np.nan, np.nan, np.nan], dtype='float64')
)
dt2a = dt2.to_arrow_table()
dt2 = vaex.from_arrow_table(dt2a)
print(dt2.dtypes)

dt2['val'] = dt2['val'].astype('string')
print(dt2.dtypes)  # errors here
```

<details>
<summary>Traceback</summary>

```
---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
~\Miniconda3\envs\api\lib\site-packages\vaex\scopes.py in evaluate(self, expression, out)
    101             # logger.debug("try avoid evaluating: %s", expression)
--> 102             result = self[expression]
    103         except KeyError:

~\Miniconda3\envs\api\lib\site-packages\vaex\scopes.py in __getitem__(self, variable)
    160             if variable not in self.values:
--> 161                 raise KeyError("Unknown variables or column: %r" % (variable,))
    162 

KeyError: 'Unknown variables or column: "astype(__val, \'str\')"'

During handling of the above exception, another exception occurred:

AttributeError                            Traceback (most recent call last)
~\Miniconda3\envs\api\lib\site-packages\vaex\dataframe.py in data_type(self, expression, array_type, internal)
   1960             try:
-> 1961                 data = self.evaluate(expression, 0, 1, filtered=False, array_type=array_type, parallel=False)
   1962             except:

~\Miniconda3\envs\api\lib\site-packages\vaex\dataframe.py in evaluate(self, expression, i1, i2, out, selection, filtered, array_type, parallel, chunk_size)
   2607         else:
-> 2608             return self._evaluate_implementation(expression, i1=i1, i2=i2, out=out, selection=selection, filtered=filtered, array_type=array_type, parallel=parallel, chunk_size=chunk_size)
   2609 

~\Miniconda3\envs\api\lib\site-packages\vaex\dataframe.py in _evaluate_implementation(self, expression, i1, i2, out, selection, filtered, array_type, parallel, chunk_size, raw)
   5551                     scope.buffers[expression] = out
-> 5552                 value = scope.evaluate(expression)
   5553                 # if isinstance(value, ColumnString) and not internal:

~\Miniconda3\envs\api\lib\site-packages\vaex\scopes.py in evaluate(self, expression, out)
    101             # logger.debug("try avoid evaluating: %s", expression)
--> 102             result = self[expression]
    103         except KeyError:

~\Miniconda3\envs\api\lib\site-packages\vaex\scopes.py in __getitem__(self, variable)
    152                     # self._ensure_buffer(variable)
--> 153                     values = self.evaluate(expression)
    154                     values = wrap(values)

~\Miniconda3\envs\api\lib\site-packages\vaex\scopes.py in evaluate(self, expression, out)
    107             # eval("def f(")
--> 108             result = eval(expression, expression_namespace, self)
    109             self.values[expression] = result

<string> in <module>

~\Miniconda3\envs\api\lib\site-packages\vaex\arrow\numpy_dispatch.py in wrapper(*args, **kwargs)
    161         kwargs = {k: unwrap(v) for k, v, in kwargs.items()}
--> 162         result = f(*args, **kwargs)
    163         return wrap(result)

~\Miniconda3\envs\api\lib\site-packages\vaex\functions.py in _astype(x, dtype)
   2319                 pass  # does not work for all data
-> 2320         return _to_string_column(x)
   2321     # we rely on numpy for astype conversions (TODO: possible performance hit?)

~\Miniconda3\envs\api\lib\site-packages\vaex\column.py in _to_string_column(x, force)
    463 def _to_string_column(x, force=True):
--> 464     ss = _to_string_sequence(x, force=force)
    465     if not isinstance(ss, (vaex.strings.StringList64, vaex.strings.StringList32)):

~\Miniconda3\envs\api\lib\site-packages\vaex\column.py in _to_string_sequence(x, force)
    435         from vaex.arrow import convert
--> 436         return convert.column_from_arrow_array(x).string_sequence
    437     elif isinstance(x, np.ndarray):

AttributeError: 'numpy.ndarray' object has no attribute 'string_sequence'

During handling of the above exception, another exception occurred:

KeyError                                  Traceback (most recent call last)
~\Miniconda3\envs\api\lib\site-packages\vaex\scopes.py in evaluate(self, expression, out)
    101             # logger.debug("try avoid evaluating: %s", expression)
--> 102             result = self[expression]
    103         except KeyError:

~\Miniconda3\envs\api\lib\site-packages\vaex\scopes.py in __getitem__(self, variable)
    160             if variable not in self.values:
--> 161                 raise KeyError("Unknown variables or column: %r" % (variable,))
    162 

KeyError: 'Unknown variables or column: "astype(__val, \'str\')"'

During handling of the above exception, another exception occurred:

AttributeError                            Traceback (most recent call last)
<ipython-input-24-10f385214ca7> in <module>
      7 
      8 dt2['val'] = dt2['val'].astype('string')
----> 9 print(dt2.dtypes)

~\Miniconda3\envs\api\lib\site-packages\vaex\dataframe.py in __getattr__(self, name)
    200             return self.__hidden__[name].__get__(self)
    201         else:
--> 202             return object.__getattribute__(self, name)
    203 
    204     def _ipython_key_completions_(self):

~\Miniconda3\envs\api\lib\site-packages\vaex\dataframe.py in dtypes(self)
   1994         """Gives a Pandas series object containing all numpy dtypes of all columns (except hidden)."""
   1995         from pandas import Series
-> 1996         return Series({column_name:self.data_type(column_name) for column_name in self.get_column_names()})
   1997 
   1998     def schema(self):

~\Miniconda3\envs\api\lib\site-packages\vaex\dataframe.py in <dictcomp>(.0)
   1994         """Gives a Pandas series object containing all numpy dtypes of all columns (except hidden)."""
   1995         from pandas import Series
-> 1996         return Series({column_name:self.data_type(column_name) for column_name in self.get_column_names()})
   1997 
   1998     def schema(self):

~\Miniconda3\envs\api\lib\site-packages\vaex\dataframe.py in data_type(self, expression, array_type, internal)
   1961                 data = self.evaluate(expression, 0, 1, filtered=False, array_type=array_type, parallel=False)
   1962             except:
-> 1963                 data = self.evaluate(expression, 0, 1, filtered=True, array_type=array_type, parallel=False)
   1964         if data_type is None:
   1965             # means we have to determine it from the data

~\Miniconda3\envs\api\lib\site-packages\vaex\dataframe.py in evaluate(self, expression, i1, i2, out, selection, filtered, array_type, parallel, chunk_size)
   2606             return self.evaluate_iterator(expression, s1=i1, s2=i2, out=out, selection=selection, filtered=filtered, array_type=array_type, parallel=parallel, chunk_size=chunk_size)
   2607         else:
-> 2608             return self._evaluate_implementation(expression, i1=i1, i2=i2, out=out, selection=selection, filtered=filtered, array_type=array_type, parallel=parallel, chunk_size=chunk_size)
   2609 
   2610     def evaluate_iterator(self, expression, s1=None, s2=None, out=None, selection=None, filtered=True, array_type=None, parallel=True, chunk_size=None, prefetch=True):

~\Miniconda3\envs\api\lib\site-packages\vaex\dataframe.py in _evaluate_implementation(self, expression, i1, i2, out, selection, filtered, array_type, parallel, chunk_size, raw)
   5550                 if out is not None:
   5551                     scope.buffers[expression] = out
-> 5552                 value = scope.evaluate(expression)
   5553                 # if isinstance(value, ColumnString) and not internal:
   5554                 #     value = value.to_numpy()

~\Miniconda3\envs\api\lib\site-packages\vaex\scopes.py in evaluate(self, expression, out)
    100         try:
    101             # logger.debug("try avoid evaluating: %s", expression)
--> 102             result = self[expression]
    103         except KeyError:
    104             # logger.debug("no luck, eval: %s", expression)

~\Miniconda3\envs\api\lib\site-packages\vaex\scopes.py in __getitem__(self, variable)
    151                 else:
    152                     # self._ensure_buffer(variable)
--> 153                     values = self.evaluate(expression)
    154                     values = wrap(values)
    155                     self.values[variable] = values

~\Miniconda3\envs\api\lib\site-packages\vaex\scopes.py in evaluate(self, expression, out)
    106             # logger.debug("in eval")
    107             # eval("def f(")
--> 108             result = eval(expression, expression_namespace, self)
    109             self.values[expression] = result
    110             # if out is not None:

<string> in <module>

~\Miniconda3\envs\api\lib\site-packages\vaex\arrow\numpy_dispatch.py in wrapper(*args, **kwargs)
    160         args = list(map(unwrap, args))
    161         kwargs = {k: unwrap(v) for k, v, in kwargs.items()}
--> 162         result = f(*args, **kwargs)
    163         return wrap(result)
    164     return wrapper

~\Miniconda3\envs\api\lib\site-packages\vaex\functions.py in _astype(x, dtype)
   2318             except TypeError:
   2319                 pass  # does not work for all data
-> 2320         return _to_string_column(x)
   2321     # we rely on numpy for astype conversions (TODO: possible performance hit?)
   2322     if isinstance(x, vaex.column.ColumnString):

~\Miniconda3\envs\api\lib\site-packages\vaex\column.py in _to_string_column(x, force)
    462 
    463 def _to_string_column(x, force=True):
--> 464     ss = _to_string_sequence(x, force=force)
    465     if not isinstance(ss, (vaex.strings.StringList64, vaex.strings.StringList32)):
    466         ss = ss.to_arrow()

~\Miniconda3\envs\api\lib\site-packages\vaex\column.py in _to_string_sequence(x, force)
    434     elif isinstance(x, pa.Array):
    435         from vaex.arrow import convert
--> 436         return convert.column_from_arrow_array(x).string_sequence
    437     elif isinstance(x, np.ndarray):
    438         mask = None

AttributeError: 'numpy.ndarray' object has no attribute 'string_sequence'
```
</details>
Fixes #1118
For discussion around the issue as well as a reproducible example, please look here: https://github.com/vaexio/vaex/issues/1118


Discussion:
A possible fix is to replace the following lines in vaex_core/column.py. I'll submit a PR later today.

Before:
```
def _to_string_sequence(x, force=True):
    ...
    elif isinstance(x, pa.Array):
        from vaex.arrow import convert
        return convert.column_from_arrow_array(x).string_sequence
    ...
```

After:
```
def _to_string_sequence(x, force=True):
    ...
    elif isinstance(x, pa.Array):
        from vaex.arrow import convert
        converted = convert.column_from_arrow_array(x)
        if isinstance(converted, np.ndarray):  # Handle cases when convert.column_from_arrow_array() returns an ndarray
            return _to_string_sequence(converted.astype(str), force)
        else:
            return converted.string_sequence
    ...
```
Hey! 

Good work! Can you please include a unit-test as well (that would break without your fix). 

Thanks!
I just added a unit test that fails without the change. Please let me know if I did something incorrectly, or if you need anything else from me. Thanks again for such an awesome library!
Is there a timeline of when this might be accepted into the master branch? It's necessary for one of my use cases, and I'm trying to determine whether I should wait or manually update the affected file during my system's build step.
Yes, I plan to take a look at it this week, and probably do a release as well.

Resolving patch:
diff --git a/packages/vaex-core/vaex/functions.py b/packages/vaex-core/vaex/functions.py
--- a/packages/vaex-core/vaex/functions.py
+++ b/packages/vaex-core/vaex/functions.py
@@ -2314,24 +2314,29 @@ def _float(x):
 
 @register_function(name='astype', on_expression=False)
 def _astype(x, dtype):
-    if dtype == 'str':
-        if isinstance(x, np.ndarray) and x.dtype.kind not in 'US':
-            try:
-                x = x.astype(dtype).astype('O')
-                mask = np.isnan(x)
-                x[mask] = None
-            except TypeError:
-                pass  # does not work for all data
-        return _to_string_column(x)
-    # we rely on numpy for astype conversions (TODO: possible performance hit?)
     if isinstance(x, vaex.column.ColumnString):
-        x = x.to_numpy()
+        x = x.to_arrow()
     if isinstance(x, vaex.array_types.supported_arrow_array_types):
-        x = x.to_pandas().values
-    y = x.astype(dtype if dtype not in ['string', 'large_string'] else 'str')
-    if vaex.column._is_stringy(y):
-        y = vaex.column._to_string_column(y)
-    return y
+        if pa.types.is_timestamp(x.type):
+            # arrow does not support timestamp to int/float, so we use numpy
+            y = x.to_numpy().astype(dtype)
+            return y
+
+        if dtype.startswith('datetime64'):  # parse dtype
+            if len(dtype) > len('datetime64'):
+                units = dtype[len('datetime64')+1:-1]
+            else:
+                units = 'ns'
+            dtype = pa.timestamp(units)
+
+        y = x.cast(dtype, safe=False)
+        return y
+    else:  # numpy case
+        if dtype in ['str', 'string', 'large_string']:
+            y = x.astype('str')
+            return vaex.column._to_string_column(y)
+        else:
+            return x.astype(dtype)
 
 
 @register_function(name='isin', on_expression=False)


Test patch:
diff --git a/tests/astype_test.py b/tests/astype_test.py
--- a/tests/astype_test.py
+++ b/tests/astype_test.py
@@ -8,7 +8,7 @@ def test_astype(ds_local):
     #ds.columns['x'] = (ds.columns['x']*1).copy()  # convert non non-big endian for now
     ds['x'] = ds['x'].astype('f4')
 
-    assert ds.x.evaluate().dtype == np.float32
+    assert ds.x.dtype == 'float32'
     assert ds.x.tolist() == ds_original.x.as_numpy().evaluate().astype(np.float32).tolist()
 
 
@@ -18,8 +18,20 @@ def test_astype_str():
     df['x'] = df['x'].str.replace(',', '').evaluate()
     df['x'] = (df['x'].astype('float')).astype('int64').evaluate()
 
-    assert df.columns['x'].dtype == np.int64
-    assert df.x.dtype == np.int64
+    assert df.x.dtype == int
+
+
+def test_astype_to_str(array_factory):
+    df = vaex.from_arrays(x=array_factory([1, 2, None]))
+    assert df.x.astype('str').tolist() == ['1', '2', None]
+
+
+def test_astype_numeric(array_factory):
+    df = vaex.from_arrays(x=array_factory([1, 2, None]))
+    assert df.x.astype('float32').tolist() == [1., 2., None]
+    assert df.x.astype('float64').tolist() == [1., 2., None]
+    assert df.x.astype('int8').tolist() == [1., 2., None]
+
 
 def test_astype_dtype():
     df = vaex.from_arrays(x=[0, 1])
diff --git a/tests/datetime_test.py b/tests/datetime_test.py
--- a/tests/datetime_test.py
+++ b/tests/datetime_test.py
@@ -128,7 +128,7 @@ def test_create_datetime64_column_from_str():
 
     expr = df.year + '-' + df.month + '-' + df.day + 'T' + df.hour + ':' + df.minute
     assert expr.values.astype(np.datetime64).tolist() == expr.astype('datetime64').tolist()
-    assert expr.values.astype('datetime64[ns]').tolist() == expr.astype('datetime64[ns]').tolist()
+    assert expr.values.astype('datetime64[ns]').tolist() == expr.astype('datetime64[ns]').to_numpy().tolist()
 
 def test_create_str_column_from_datetime64():
     date = np.array([np.datetime64('2009-10-12T03:31:00'),
diff --git a/tests/open_test.py b/tests/open_test.py
--- a/tests/open_test.py
+++ b/tests/open_test.py
@@ -98,7 +98,7 @@ def _cleanup_generated_files():
         os.remove(hdf5_file)
     for hdf5_file in glob.glob(os.path.join(path, 'data', 'output', '*.yaml')):
         os.remove(hdf5_file)
-    for hdf5_file in glob.glob(os.path.join(path, 'data', '*.hdf5')):
+    for hdf5_file in glob.glob(os.path.join(path, 'data', 'small*.hdf5')):
         os.remove(hdf5_file)
     for hdf5_file in glob.glob(os.path.join(path, 'data', 'output', '*.hdf5')):
         os.remove(hdf5_file)