Repository: vaexio/vaex

Issue:
[BUG-REPORT] groupby AttributeError: 'str' object has no attribute 'kind'
**Description**
Have a `numpy.ndarray` of shape `(25, 24)` that I'm adding as columns to a `df` with:
```
for i in range(len(hours_col) - 1):
    df.add_column("hour_%d" % i, hours_col.T[i], dtype='uint16')
```

It gets added fine & I see:
```
Columns:
column	type	unit	description	expression
IDN_NUM int64		
dow	virtual column	                astype(dt_dayofweek(__START_DT),` 'uint8')
hour_0	uint16		
hour_1	uint16		
hour_2	uint16		
hour_3	uint16		
hour_4	uint16		
hour_5	uint16		
hour_6	uint16		
hour_7	uint16		
hour_8	uint16		
hour_9	uint16		
hour_10	uint16		
hour_11	uint16		
hour_12	uint16		
hour_13	uint16		
hour_14	uint16		
hour_15	uint16		
hour_16	uint16		
hour_17	uint16		
hour_18	uint16		
hour_19	uint16		
hour_20	uint16		
hour_21	uint16		
hour_22	uint16		
hour_23	uint16		
```

However, when I do `type(df.hour_0.dtype)`, it says `str`.
Whereas, `type(df.dow.dtype)` says `numpy.dtype`.

`df.hour_0` says:
```
Expression = hour_0
Length: 25 dtype: uint16 (column)
---------------------------------
 0     0
 1  1777
 2     0
 3     0
 4     0
  ...   
20     0
21   122
22     0
23     0
24     0
```

So, the following `groupby` errors out:
```
df = df.groupby([df.IDN_NUM]).agg('sum')
```


Is this because `hour_%` are `expressions`, or maybe because `dow` is a `virtual column` that needs to be materialized somehow?
**EDIT:** I've just tried `df = df.materialize('dow')` and the error still remains.


**Error**
```
c:\program files\python38\lib\site-packages\vaex\agg.py in _prepare_types(self, df)
     95                 self.dtype_out = np.dtype('int64')
     96             if self.short_name in ['sum', 'summoment']:
---> 97                 self.dtype_out = vaex.utils.upcast(self.dtype_in)
     98 
     99     def add_operations(self, agg_task, **kwargs):

c:\program files\python38\lib\site-packages\vaex\utils.py in upcast(dtype)
    910 
    911 def upcast(dtype):
--> 912     if dtype.kind == "b":
    913         return np.dtype('int64')
    914     if dtype.kind == "i":

AttributeError: 'str' object has no attribute 'kind'
```


**Software information**
 - Vaex version: 
 ```{'vaex': '3.0.0',
 'vaex-core': '2.0.3',
 'vaex-viz': '0.4.0',
 'vaex-hdf5': '0.6.0',
 'vaex-server': '0.3.1',
 'vaex-astro': '0.7.0',
 'vaex-jupyter': '0.5.2',
 'vaex-ml': '0.9.0',
 'vaex-arrow': '0.5.1'
``` 
 - Vaex was installed via: pip
 - OS: WIn10



Discussion:
Any comment please, @maartenbreddels @JovanVeljanoski ?
Could you try upgrading using `pip install vaex==4.0.0a8` and see if it still happens?
Getting this instead:

```
---------------------------------------------------------------------------
ModuleNotFoundError                       Traceback (most recent call last)
<ipython-input-8-0616055ca1db> in <module>
      1 get_ipython().run_line_magic('timeit', '')
----> 2 df, rt1 = f1_read_in()
      3 df, rt2 = f2_midnight_splits(df)
      4 df, df2, hours_col, rt3a, rt3b, rt3 = f3_process_calls(df)
      5 

<ipython-input-4-fc63bfe58279> in f1_read_in()
     26 
---> 27     df = vaex.from_csv(filename,
     28                        convert=True,
     29 #                        dtypes=dtypes,  # dtypes not working for internet file

c:\program files\python38\lib\site-packages\vaex\__init__.py in from_csv(filename_or_buffer, copy_index, chunk_size, convert, fs_options, fs, **kwargs)
    521         import vaex.convert
    522         path_output = convert if isinstance(convert, str) else vaex.convert._convert_name(filename_or_buffer)
--> 523         vaex.convert.convert_csv(
    524             path_input=filename_or_buffer, fs_options_input=fs_options, fs_input=fs,
    525             path_output=path_output, fs_options_output=fs_options, fs_output=fs,

c:\program files\python38\lib\site-packages\vaex\convert.py in convert_csv(path_input, fs_options_input, fs_input, path_output, fs_options_output, fs_output, *args, **kwargs)
     49             kwargs['chunk_size'] = 5_000_000
     50         _from_csv_convert_and_read(path_input, maybe_convert_path=path_output, fs_options=fs_options_input, fs=fs_input, **kwargs)
---> 51     cached_output(*args, **kwargs)
     52 
     53 

c:\program files\python38\lib\site-packages\vaex\cache.py in call(callable, *args, **kwargs)
     52                 if not vaex.file.exists(path_output_meta, fs_options=fs_options_output_, fs=fs_output):
     53                     log.info('file including fingerprint not found (%) or does not exist yet, running conversion %s → %s', path_output_meta, path_input, path_output)
---> 54                     value = callable(*args, **kwargs)
     55                     write_fingerprint()
     56                     return value

c:\program files\python38\lib\site-packages\vaex\convert.py in cached_output(*args, **kwargs)
     48             # make it memory efficient by default
     49             kwargs['chunk_size'] = 5_000_000
---> 50         _from_csv_convert_and_read(path_input, maybe_convert_path=path_output, fs_options=fs_options_input, fs=fs_input, **kwargs)
     51     cached_output(*args, **kwargs)
     52 

c:\program files\python38\lib\site-packages\vaex\convert.py in _from_csv_convert_and_read(filename_or_buffer, maybe_convert_path, chunk_size, fs_options, fs, copy_index, **kwargs)
     72             df = vaex.from_pandas(df_pandas, copy_index=copy_index)
     73             filename_hdf5 = _convert_name(csv_path, suffix='_chunk%d' % i)
---> 74             df.export_hdf5(filename_hdf5)
     75             converted_paths.append(filename_hdf5)
     76             log.info('saved chunk #%d to %s' % (i, filename_hdf5))

c:\program files\python38\lib\site-packages\vaex\dataframe.py in export_hdf5(self, path, byteorder, progress, chunk_size, parallel, column_count, writer_threads)
   6067         :return:
   6068         """
-> 6069         from vaex.hdf5.writer import Writer
   6070         with Writer(path) as writer:
   6071             writer.write(self, chunk_size=chunk_size, progress=progress, column_count=column_count)

ModuleNotFoundError: No module named 'vaex.hdf5.writer'
```
Strange, what does `pip show vaex-hdf5` say?
```
C:\Users\bzzzz>pip show vaex-hdf5
Name: vaex-hdf5
Version: 0.7.0a6
Summary: hdf5 file support for vaex
Home-page: https://www.github.com/maartenbreddels/vaex
Author: Maarten A. Breddels
Author-email: maartenbreddels@gmail.com
License: MIT
Location: c:\users\bzzzz\appdata\roaming\python\python38\site-packages
Requires: vaex-core, h5py
Required-by: vaex
```

For now, I've bypassed this by commenting out `convert=True,` in `vaex.from_csv`.  
  
  
However, `groupby` is aggregating one of the columns I'm grouping by.
**EDIT**: `groupby` works fine when I reran the whole thing from scratch.

It also seems to have changed the dtypes.
IDN_NUM should remain `uint64` (was changed to `int32`).
hour_x_sum should remain `uint16` (was changed to `uint64`).

```
column	type	unit	description	expression
IDN_NUM	int32		
dow_sum	uint64		
hour_0_sum	uint64		
hour_1_sum	uint64		
hour_2_sum	uint64		
hour_3_sum	uint64		
hour_4_sum	uint64		
hour_5_sum	uint64		
hour_6_sum	uint64		
hour_7_sum	uint64		
hour_8_sum	uint64		
hour_9_sum	uint64		
hour_10_sum	uint64		
hour_11_sum	uint64		
hour_12_sum	uint64		
hour_13_sum	uint64		
hour_14_sum	uint64		
hour_15_sum	uint64		
hour_16_sum	uint64		
hour_17_sum	uint64		
hour_18_sum	uint64		
hour_19_sum	uint64		
hour_20_sum	uint64		
hour_21_sum	uint64		
hour_22_sum	uint64
hour_23_sum	uint64	
```
> hour_x_sum should remain `uint16` (was changed to `uint64`).

Yes, it seems we ignore dtype. I'm not sure we should support it, and just remove it. Vaex does not copy your data, it only will keep a reference, so it's using the original uint64.




> IDN_NUM should remain `uint64` (was changed to `int32`).

I'm not sure where this changes, but when you do groupby, a new column is created, I think it makes sense to make this is small as possible to save memory right?
Thank you & everyone who has worked on this incredible library & for sharing!    
    



> > hour_x_sum should remain `uint16` (was changed to `uint64`).
> 
> Yes, it seems we ignore dtype. I'm not sure we should support it, and just remove it. Vaex does not copy your data, it only will keep a reference, so it's using the original uint64.  

Sorry, please clarify. "just remove it" refers to... ? 
Do you mean in the `add_column` function?  

In the first post of this issue, `hour_` columns are `uint16`, so the original is/was `uint16`.
It's fine to upcast but not to the max possible because with millions of rows & 24 columns, their size adds up on disk.  
  
  
> > IDN_NUM should remain `uint64` (was changed to `int32`).
> 
> I'm not sure where this changes, but when you do groupby, a new column is created, I think it makes sense to make this is small as possible to save memory right?  

Hmm.. yeah, makes sense. As long as it upcasts to `uint64` when the numbers go beyond `2,147,483,647`, it should be fine.
  
  
**EDIT**: Also, any comments on this issue - `ModuleNotFoundError: No module named 'vaex.hdf5.writer'` ?
> Thank you & everyone who has worked on this incredible library & for sharing!
> 
> > >

Thanks!



> Sorry, please clarify. "just remove it" refers to... ?

I think we should remove the dtype argument from that method, it doesn't do anything, and we should not touch the dtype.



> It's fine to upcast but not to the max possible because with millions of rows & 24 columns, their size adds up on disk.
> 
> > >

it is only in the aggregation when it is upcasted right, not in the original dataframe, e.g.

![image](https://user-images.githubusercontent.com/1765949/105144563-b7189780-5afd-11eb-978e-fbc73fef5d7c.png)

This has to be done, because in the sum aggregation we don't want to have integer overflow.

I don't get the int32 for IDN_NUM, I think that might be Windows behavior for NumPy.
> it is only in the aggregation when it is upcasted right, not in the original dataframe, e.g.  

Yes, the original df is fine, but I'll be replacing the original with the groupby'd aggregated df, i.e., 
`df = df.groupby([df.IDN_NUM]).agg('sum')`  
so the output df will be far bigger than the input df; might cause problems keeping it in memory before adding a few more rows & writing it out to a file.



Any comment on the hdf5 issue from a previous reply? 
https://github.com/vaexio/vaex/issues/1165#issuecomment-762915929

I'm inching closer to a fully working program :)

Resolving patch:
diff --git a/packages/vaex-core/vaex/convert.py b/packages/vaex-core/vaex/convert.py
--- a/packages/vaex-core/vaex/convert.py
+++ b/packages/vaex-core/vaex/convert.py
@@ -47,19 +47,14 @@ def cached_output(*args, **kwargs):
         if 'chunk_size' not in kwargs:
             # make it memory efficient by default
             kwargs['chunk_size'] = 5_000_000
-        _from_csv_convert_and_read(path_input, maybe_convert_path=path_output, fs_options=fs_options_input, fs=fs_input, **kwargs)
+        _from_csv_convert_and_read(path_input, path_output=path_output, fs_options=fs_options_input, fs=fs_input, **kwargs)
     cached_output(*args, **kwargs)
 
 
-def _from_csv_convert_and_read(filename_or_buffer, maybe_convert_path, chunk_size, fs_options, fs=None, copy_index=False, **kwargs):
+def _from_csv_convert_and_read(filename_or_buffer, path_output, chunk_size, fs_options, fs=None, copy_index=False, **kwargs):
     # figure out the CSV file path
-    if isinstance(maybe_convert_path, str):
-        csv_path = re.sub(r'\.hdf5$', '', str(maybe_convert_path), flags=re.IGNORECASE)
-    elif isinstance(filename_or_buffer, str):
-        csv_path = filename_or_buffer
-    else:
-        raise ValueError('Cannot derive filename to use for converted HDF5 file, '
-                         'please specify it using convert="my.csv.hdf5"')
+    csv_path = vaex.file.stringyfy(filename_or_buffer)
+    path_output_bare, ext, _ = vaex.file.split_ext(path_output)
 
     combined_hdf5 = _convert_name(csv_path)
 
@@ -70,20 +65,20 @@ def _from_csv_convert_and_read(filename_or_buffer, maybe_convert_path, chunk_siz
         csv_reader = pd.read_csv(filename_or_buffer, chunksize=chunk_size, **kwargs)
         for i, df_pandas in enumerate(csv_reader):
             df = vaex.from_pandas(df_pandas, copy_index=copy_index)
-            filename_hdf5 = _convert_name(csv_path, suffix='_chunk%d' % i)
-            df.export_hdf5(filename_hdf5)
-            converted_paths.append(filename_hdf5)
-            log.info('saved chunk #%d to %s' % (i, filename_hdf5))
+            chunk_name = f'{path_output_bare}_chunk_{i}.{ext}'
+            df.export(chunk_name)
+            converted_paths.append(chunk_name)
+            log.info('saved chunk #%d to %s' % (i, chunk_name))
 
     # combine chunks into one HDF5 file
     if len(converted_paths) == 1:
         # no need to merge several HDF5 files
-        os.rename(converted_paths[0], combined_hdf5)
+        os.rename(converted_paths[0], path_output)
     else:
-        log.info('converting %d chunks into single HDF5 file %s' % (len(converted_paths), combined_hdf5))
+        log.info('converting %d chunks into single file %s' % (len(converted_paths), path_output))
         dfs = [vaex.open(p) for p in converted_paths]
         df_combined = vaex.concat(dfs)
-        df_combined.export_hdf5(combined_hdf5)
+        df_combined.export(path_output)
 
         log.info('deleting %d chunk files' % len(converted_paths))
         for df, df_path in zip(dfs, converted_paths):
@@ -91,8 +86,7 @@ def _from_csv_convert_and_read(filename_or_buffer, maybe_convert_path, chunk_siz
                 df.close()
                 os.remove(df_path)
             except Exception as e:
-                log.error('Could not close or delete intermediate hdf5 file %s used to convert %s to hdf5: %s' % (
-                    df_path, csv_path, e))
+                log.error('Could not close or delete intermediate file %s used to convert %s to single file: %s', (df_path, csv_path, path_output))
 
 
 def main(argv):


Test patch:
diff --git a/tests/open_test.py b/tests/open_test.py
--- a/tests/open_test.py
+++ b/tests/open_test.py
@@ -32,6 +32,11 @@ def test_open_convert_kwargs():
     assert df.x.dtype == 'float64'
 
 
+def test_open_convert_non_hdf5():
+    csv2 = os.path.join(path, 'data', 'small2.csv')
+    df = vaex.open(csv2, convert='small2.parquet')
+
+
 def test_open_convert_explicit_path():
     # explicit path, also a truthy
     if not os.path.exists(target_path):