Repository: vaexio/vaex

Issue:
concatenate with masked arrays
```python
import vaex
import numpy as np
df1 = vaex.from_arrays(x=np.arange(3.))
df2 = vaex.from_arrays(x=np.ma.masked_all(4, dtype='f8'))
df1.concat(df2)
```
Gives:
```
~/src/vaex/packages/vaex-core/vaex/column.py in __init__(self, expressions, dtype)
    145                             self.fill_value = expressions[0][0:1].fill_value
    146                         except:
--> 147                             self.fill_value = expressions[0].values.fill_value
    148 
    149             any_strings = any([dtype == str_type for dtype in dtypes])

AttributeError: 'numpy.ndarray' object has no attribute 'fill_value'
```

More strangely:
```
import vaex
import numpy as np

@vaex.register_dataframe_accessor('mytool', override=True)
class mytool:
    def __init__(self, df):
        self.df = df

    def shift(self, column, n, cyclic=True, fill_value=None, inplace=False):
        # make a copy without column
        df = self.df.copy().drop(column)
        # make a copy with just the colum
        df_column = self.df[[column]]
        # slice off the head and tail
        if cyclic:
            df_head = df_column[-n:]
        else:
            df_head = vaex.from_dict({column: np.ma.masked_all(n, dtype=self.df[column].dtype)})
        df_tail = df_column[:-n]
        # stitch them together
        df_shifted = df_head.concat(df_tail)
        # and join (based on row number)
        return df.join(df_shifted, inplace=inplace)

x = np.arange(10)
y = x**2
df = vaex.from_arrays(x=x, y=y)
df['shifted_y'] = df.y
df2 = df.mytool.shift('shifted_y', 2, cyclic=False)
df2
```
This silently does not include the dropped column, (a non-cyclic version of #660 )


Discussion:

Resolving patch:
diff --git a/packages/vaex-core/vaex/column.py b/packages/vaex-core/vaex/column.py
--- a/packages/vaex-core/vaex/column.py
+++ b/packages/vaex-core/vaex/column.py
@@ -135,16 +135,22 @@ def __getitem__(self, slice):
 
 class ColumnConcatenatedLazy(Column):
     def __init__(self, expressions, dtype=None):
+        self.is_masked = any([e.is_masked for e in expressions])
+        if self.is_masked:
+            for expression in expressions:
+                if expression.is_masked:
+                    try:
+                        # fast path
+                        self.fill_value = expression[0:1].fill_value
+                        break
+                    except:  # noqa
+                        # slower path (we have to evaluate everything)
+                        self.fill_value = expression.values.fill_value
+                        break
+            else:
+                raise ValueError('Concatenating expressions with masked values, but no fill value is found')
         if dtype is None:
             dtypes = [e.dtype for e in expressions]
-            self.is_masked = any([e.is_masked for e in expressions])
-            if self.is_masked:
-                for expression in expressions:
-                    if expression.is_masked:
-                        try:
-                            self.fill_value = expressions[0][0:1].fill_value
-                        except:
-                            self.fill_value = expressions[0].values.fill_value
 
             any_strings = any([dtype == str_type for dtype in dtypes])
             if any_strings:
@@ -226,7 +232,7 @@ def __getitem__(self, slice):
                 warnings.warn("might be slow, you have concatenated expressions with a filter set")
             values = current_expression.evaluate(i1=start - offset, i2=stop - offset, parallel=False)
         else:
-            if current_expression.is_masked:
+            if self.is_masked:
                 copy = np.ma.empty(stop - start, dtype=dtype)
                 copy.fill_value = self.fill_value
             else:
diff --git a/packages/vaex-core/vaex/dataframe.py b/packages/vaex-core/vaex/dataframe.py
--- a/packages/vaex-core/vaex/dataframe.py
+++ b/packages/vaex-core/vaex/dataframe.py
@@ -5883,7 +5883,6 @@ def __init__(self, dfs, name=None):
         for name in list(first.virtual_columns.keys()):
             if all([first.virtual_columns[name] == df.virtual_columns.get(name, None) for df in tail]):
                 self.add_virtual_column(name, first.virtual_columns[name])
-                self.column_names.append(name)
             else:
                 self.columns[name] = ColumnConcatenatedLazy([df[name] for df in dfs])
                 self.column_names.append(name)


Test patch:
diff --git a/tests/concat_test.py b/tests/concat_test.py
--- a/tests/concat_test.py
+++ b/tests/concat_test.py
@@ -106,6 +106,31 @@ def test_sliced_concat(i1, length, df_concat):
     assert df.x.tolist() == x[i1:i2]
 
 
+def test_concat_masked_values(df_concat):
+    df = df_concat
+    # evaluate a piece not containing masked values
+    assert df.m.evaluate(0, 3).tolist() == df.x[:3].tolist()
+
+
+def test_concat_masked_with_unmasked():
+    # Exposes https://github.com/vaexio/vaex/issues/661
+    df1 = vaex.from_arrays(x=np.arange(3.))
+    df2 = vaex.from_arrays(x=np.ma.masked_all(2, dtype='f8'))
+    df = df1.concat(df2)
+    assert df.x.tolist() == [0, 1, 2, None, None]
+
+
+def test_concat_virtual_column_names():
+    df1 = vaex.from_arrays(x=np.arange(3.))
+    df2 = vaex.from_arrays(x=np.arange(4.))
+    df1['z'] = df1.x ** 2
+    df2['z'] = df2.x ** 2
+    df = df1.concat(df2)
+    # test that we get the correct column names (we had duplicates before)
+    assert df.get_column_names() == ['x', 'z']
+    assert list(df) == ['x', 'z']
+
+
 def test_concat_missing_values():
     df1 = vaex.from_arrays(x=[1, 2, 3], y=[np.nan, 'b', 'c'])
     df2 = vaex.from_arrays(x=[4, 5, np.nan], y=['d', 'e', 'f'])