Repository: vaexio/vaex

Issue:
[BUG-REPORT] export_hdf5 fails after type change from string to datetime
**Description**
Exporting the dataframe to hdf5 fails after changing the column type from string to datetime64.
The `date` column is in the form of '2020-11-25'.

```
df = vaex.open('df.hdf5')

df['current_date'] = df['date'].astype('datetime64')

df.export_hdf5('df_final.hdf5', progress=True)
```

**Software information**
 - Vaex version (`import vaex; vaex.__version__)`:
vaex==4.0.0a4
vaex-arrow==0.5.1
vaex-astro==0.8.0.dev1
vaex-core==4.0.0a9
vaex-hdf5==0.7.0a4
vaex-jupyter==0.6.0.dev1
vaex-ml==0.11.0a4
vaex-server==0.4.0.dev1
vaex-viz==0.5.0.dev1
 - Vaex was installed via: pip 
 - OS: Ubuntu 20.04

**Additional information**

Below is the output:

[##################################------] 85.71% estimated time:     0.47s =  0.0m =  0.0h 
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-112-5b4724d8d110> in <module>
     23 # df_daily_final = df_daily_final.join(vaex.open('../storage/df_daily_clean.hdf5')[['id', 'close']], left_on='six_months_later', right_on='id', rsuffix='_later', how='left')
     24 # df_daily_final.dtypes
---> 25 df_daily_final.export_hdf5('../storage/df_daily_final.hdf5', progress=True)

~/Documents/Candor/investing_etl_python/venv/lib/python3.8/site-packages/vaex/dataframe.py in export_hdf5(self, path, byteorder, progress, parallel)
   5978         """
   5979         import vaex.export
-> 5980         vaex.export.export_hdf5(self, path, byteorder=byteorder, progress=progress, parallel=parallel)
   5981 
   5982     @docsubst

~/Documents/Candor/investing_etl_python/venv/lib/python3.8/site-packages/vaex/export.py in export_hdf5(dataset, path, column_names, byteorder, shuffle, selection, progress, virtual, sort, ascending, parallel)
    252     kwargs = locals()
    253     import vaex.hdf5.export
--> 254     vaex.hdf5.export.export_hdf5(**kwargs)
    255 
    256 

~/Documents/Candor/investing_etl_python/venv/lib/python3.8/site-packages/vaex/hdf5/export.py in export_hdf5(dataset, path, column_names, byteorder, shuffle, selection, progress, virtual, sort, ascending, parallel)
    241     df_output = vaex.dataframe.DataFrameLocal(dataset_output)
    242 
--> 243     column_names = vaex.export._export(dataset_input=dataset, dataset_output=df_output, path=path, random_index_column=random_index_name,
    244                                        column_names=column_names, selection=selection, shuffle=shuffle, byteorder=byteorder,
    245                                        progress=progress, sort=sort, ascending=ascending, parallel=parallel)

~/Documents/Candor/investing_etl_python/venv/lib/python3.8/site-packages/vaex/export.py in _export(dataset_input, dataset_output, random_index_column, path, column_names, byteorder, shuffle, selection, progress, virtual, sort, ascending, parallel)
    139         for future in futures:
    140             try:
--> 141                 future.result(0.1/4)
    142             except concurrent.futures.TimeoutError:
    143                 done = False

/usr/lib/python3.8/concurrent/futures/_base.py in result(self, timeout)
    437                 raise CancelledError()
    438             elif self._state == FINISHED:
--> 439                 return self.__get_result()
    440             else:
    441                 raise TimeoutError()

/usr/lib/python3.8/concurrent/futures/_base.py in __get_result(self)
    386     def __get_result(self):
    387         if self._exception:
--> 388             raise self._exception
    389         else:
    390             return self._result

/usr/lib/python3.8/concurrent/futures/thread.py in run(self)
     55 
     56         try:
---> 57             result = self.fn(*self.args, **self.kwargs)
     58         except BaseException as exc:
     59             self.future.set_exception(exc)

~/Documents/Candor/investing_etl_python/venv/lib/python3.8/site-packages/vaex/export.py in _export_column(dataset_input, dataset_output, column_name, shuffle, sort, selection, N, order_array, order_array_inverse, progress_status, parallel)
    209                             target_set_item = slice(to_offset, to_offset + no_values)
    210                         if dtype.is_datetime:
--> 211                             values = values.view(np.int64)
    212                         if np.ma.isMaskedArray(to_array) and np.ma.isMaskedArray(values):
    213                             to_array.data[target_set_item] = values.filled(fill_value)

~/Documents/Candor/investing_etl_python/venv/lib/python3.8/site-packages/numpy/core/_internal.py in _view_is_safe(oldtype, newtype)
    457 
    458     if newtype.hasobject or oldtype.hasobject:
--> 459         raise TypeError("Cannot change data-type for object array.")
    460     return
    461 

TypeError: Cannot change data-type for object array.


Discussion:
Hi Joey,

Interesting, I cannot seem to reproduce it:
```
import vaex
df = vaex.from_arrays(date=['2020-11-25', None])
df['current_date'] = df['date'].astype('datetime64')
df.export_hdf5('1083.hdf5', progress=True)
```

For some reason, it seems to go through the dtype=object, but that does not happen in my case, only in yours. Could you try to make a reproducible example for me? 

Regards,

Maarten
Hi Maarten,

The first row that has a problem is 1025 but am unable to replicate the error manually when using those text inputs.  I also tried switching between arrow and hdf5 to see if that was the problem.  I got a different error but no ability to write the file.  

Here is a link to a sample of the data - https://gist.github.com/joeybellerose/356ffb6ee7e2d8393cab466bff8130cc

Downgrading to Vaex 3.0 solved the problem....although the reason I went to 4.0.0a4 was to solve some of the join issues I was running into.

Thanks,

Joey






 
Great, that file helped me reproduce it! Hope to be able to fix it soon.
Excellent!  

Resolving patch:
diff --git a/packages/vaex-core/vaex/array_types.py b/packages/vaex-core/vaex/array_types.py
--- a/packages/vaex-core/vaex/array_types.py
+++ b/packages/vaex-core/vaex/array_types.py
@@ -115,11 +115,15 @@ def convert(x, type, default_type="numpy"):
     import vaex.column
     if type == "numpy":
         if isinstance(x, (list, tuple)):
-            return np.concatenate([convert(k, type) for k in x])
+            return concat([convert(k, type) for k in x])
         else:
             return to_numpy(x, strict=True)
     if type == "numpy-arrow":  # used internally, numpy if possible, otherwise arrow
-        return to_numpy(x, strict=False)
+        if isinstance(x, (list, tuple)):
+            return concat([convert(k, type) for k in x])
+        else:
+            return to_numpy(x, strict=False)
+
     elif type == "arrow":
         if isinstance(x, (list, tuple)):
             return pa.chunked_array([convert(k, type) for k in x])
diff --git a/packages/vaex-core/vaex/dataframe.py b/packages/vaex-core/vaex/dataframe.py
--- a/packages/vaex-core/vaex/dataframe.py
+++ b/packages/vaex-core/vaex/dataframe.py
@@ -5438,7 +5438,7 @@ def _evaluate_implementation(self, expression, i1=None, i2=None, out=None, selec
             expression_to_evaluate = list(set(expressions))  # lets assume we have to do them all
 
             for expression in set(expressions):
-                dtypes[expression] = dtype = df.data_type(expression)
+                dtypes[expression] = dtype = df.data_type(expression).internal
                 if expression not in df.columns:
                     virtual.add(expression)
                 # since we will use pre_filter=True, we'll get chunks of the data at unknown offset


Test patch:
diff --git a/tests/evaluate_test.py b/tests/evaluate_test.py
--- a/tests/evaluate_test.py
+++ b/tests/evaluate_test.py
@@ -6,12 +6,20 @@
 @pytest.mark.parametrize("parallel", [True, False])
 def test_evaluate_iterator(df_local, chunk_size, prefetch, parallel):
     df = df_local
-    x = df.x.to_numpy()
-    total = 0
-    for i1, i2, chunk in df_local.evaluate_iterator('x', chunk_size=chunk_size, prefetch=prefetch, parallel=parallel, array_type='numpy'):
-        assert x[i1:i2].tolist() == chunk.tolist()
-        total += chunk.sum()
-    assert total == x.sum()
+    with small_buffer(df):
+        x = df.x.to_numpy()
+        z = df.z.to_numpy()
+        total = 0
+        for i1, i2, chunk in df_local.evaluate_iterator('x', chunk_size=chunk_size, prefetch=prefetch, parallel=parallel, array_type='numpy-arrow'):
+            assert x[i1:i2].tolist() == chunk.tolist()
+            total += chunk.sum()
+        assert total == x.sum()
+
+        total = 0
+        for i1, i2, chunk in df_local.evaluate_iterator('z', chunk_size=chunk_size, prefetch=prefetch, parallel=parallel, array_type='numpy-arrow'):
+            assert z[i1:i2].tolist() == chunk.tolist()
+            total += chunk.sum()
+        assert total == z.sum()
 
 
 @pytest.mark.parametrize("chunk_size", [2, 5])