Repository: vaexio/vaex

Issue:
fillna crashes
```
import vaex as vx
ds = vx.open('../datasets/train.csv')
ds['Cabin'] = ds['Cabin'].fillna('NA')
ds['Cabin']
```

Result:
```
<vaex.expression.Expression(expressions='Cabin')> instance at 0x11454beb8 values=[Error evaluating: NameError("name 'NA' is not defined",)] 
```

[dataset](https://www.kaggle.com/c/titanic/data)


Discussion:
It is exposing a bug, you can try this workaround for now:
```
ds = vaex.open('../train.csv')
ds.columns['Cabin'] = ds.columns['Cabin'].astype(str)
ds['Cabin'] = ds.apply(lambda x: 'missing' if x == 'nan' else x, ['Cabin'])
```

Resolving patch:
diff --git a/packages/vaex-core/vaex/dataset.py b/packages/vaex-core/vaex/dataset.py
--- a/packages/vaex-core/vaex/dataset.py
+++ b/packages/vaex-core/vaex/dataset.py
@@ -1047,10 +1047,17 @@ def selection_from_dict(values):
 
 
 def fillna(ar, value, fill_nan=True, fill_masked=True):
-    '''Returns an array where missing values are replaced by value
+    '''Returns an array where missing values are replaced by value.
 
+    If the dtype is object, nan values and 'nan' string values
+    are replaced by value when fill_nan==True.
     '''
-    if ar.dtype.kind == 'f' and fill_nan:
+    if ar.dtype.kind in 'O' and fill_nan:
+        strings = ar.astype(str)
+        mask = strings == 'nan'
+        ar = ar.copy()
+        ar[mask] = value
+    elif ar.dtype.kind in 'f' and fill_nan:
         mask = np.isnan(ar)
         if np.any(mask):
             ar = ar.copy()
@@ -4804,6 +4811,11 @@ def __delitem__(self, item):
             raise KeyError('no such column or virtual_columns named %r' % name)
 
     def drop(self, columns, inplace=False):
+        """Drop columns (or single column)
+
+        :param columns: list of columns or single column name
+        """
+        columns = _ensure_list(columns)
         ds = self if inplace else self.copy()
         for column in columns:
             del ds[column]
@@ -4942,7 +4954,7 @@ def myrepr(k):
                             return str(k)
                         else:
                             return repr(k)
-                    arg_string = ", ".join([myrepr(k) for k in args] + ['{}={}'.format(name, value) for name, value in kwargs.items()])
+                    arg_string = ", ".join([myrepr(k) for k in args] + ['{}={}'.format(name, myrepr(value)) for name, value in kwargs.items()])
                     expression = "{}({})".format(local_name, arg_string)
                     return vaex.expression.Expression(self, expression)
                 return wrap
diff --git a/packages/vaex-core/vaex/expression.py b/packages/vaex-core/vaex/expression.py
--- a/packages/vaex-core/vaex/expression.py
+++ b/packages/vaex-core/vaex/expression.py
@@ -276,8 +276,11 @@ def max(self, binby=[], limits=None, shape=default_shape, selection=False, delay
     def evaluate(self, i1=None, i2=None, out=None, selection=None):
         return self.ds.evaluate(self, i1, i2, out=out, selection=selection)
 
-    # def fillna(self, value, fill_nan=True, fill_masked=True):
-    #     return self.ds.func.fillna(self, value, fill_nan=fill_nan, fill_masked=fill_masked)
+    # TODO: it is not so elegant we need to have a custom version of this
+    # it now also misses the docstring, reconsider how the the meta class auto
+    # adds this method
+    def fillna(self, value, fill_nan=True, fill_masked=True):
+        return self.ds.func.fillna(self, value=value, fill_nan=fill_nan, fill_masked=fill_masked)
 
     def clip(self, lower=None, upper=None):
         return self.ds.func.clip(self, lower, upper)


Test patch:
diff --git a/tests/common.py b/tests/common.py
--- a/tests/common.py
+++ b/tests/common.py
@@ -37,8 +37,8 @@ def server(webserver):
     server.close()
 
 @pytest.fixture()
-def ds_remote(webserver, server):
-    ds = ds_trimmed()
+def ds_remote(webserver, server, ds_trimmed):
+    ds = ds_trimmed
     ds.name = 'ds_trimmed'
     webserver.set_datasets([ds])
     return server.datasets(as_dict=True)['ds_trimmed']
@@ -93,7 +93,7 @@ def create_base_ds():
     dataset.add_column("x", x)
     dataset.add_column("y", y)
     # m = x.copy()
-    m = m = np.arange(-2, 40, dtype=">f8").reshape((-1,21)).T.copy()[:,0]
+    m = np.arange(-2, 40, dtype=">f8").reshape((-1,21)).T.copy()[:,0]
     ma_value = 77777
     m[-1+10] = ma_value
     m[-1+20] = ma_value
@@ -110,16 +110,22 @@ def create_base_ds():
     nm[-1+20] = ma_value
     nm = np.ma.array(nm, mask=nm==ma_value)
 
-    mi = mi = np.ma.array(m.data.astype(np.int64), mask=m.data==ma_value, fill_value=88888)
+    mi = np.ma.array(m.data.astype(np.int64), mask=m.data==ma_value, fill_value=88888)
     dataset.add_column("m", m)
     dataset.add_column('n', n)
     dataset.add_column('nm', nm)
     dataset.add_column("mi", mi)
     dataset.add_column("ints", ints)
 
-
     name = np.array(list(map(lambda x: str(x) + "bla" + ('_' * int(x)), x)), dtype='S') #, dtype=np.string_)
     dataset.add_column("name", np.array(name))
+
+    obj_data = np.array(['train', 'false' , True, 1, 30., np.nan, 'something', 'something a bit longer resembling a sentence?!', -10000, 'this should be masked'], dtype='object')
+    obj_mask = np.array([False] * 9 + [True])
+    obj = nm.copy().astype('object')
+    obj[2:12] = np.ma.MaskedArray(data=obj_data, mask=obj_mask, dtype='object')
+    dataset.add_column("obj", obj)
+
     return dataset
 
 # dsf = create_filtered()
\ No newline at end of file
diff --git a/tests/export_test.py b/tests/export_test.py
--- a/tests/export_test.py
+++ b/tests/export_test.py
@@ -2,6 +2,8 @@
 
 def test_export(ds_local, tmpdir):
 	ds = ds_local
+	# TODO: we eventually want to support dtype=object, but not for hdf5
+	ds = ds.drop(ds.obj)
 	path = str(tmpdir.join('test.hdf5'))
 	ds.export_hdf5(path)
 	ds = ds.sample(5)
diff --git a/tests/fillna_test.py b/tests/fillna_test.py
--- a/tests/fillna_test.py
+++ b/tests/fillna_test.py
@@ -1,20 +1,39 @@
 from common import *
-import collections
 
-def test_sample(ds_local):
+
+def test_fillna_column(ds_local):
+    ds = ds_local
+    ds['ok'] = ds['obj'].fillna(value='NA')
+    assert ds.ok.values[5] == 'NA'
+
+
+def test_fillna(ds_local):
     ds = ds_local
-    x = np.arange(10).tolist()
-    dss = ds.sample(frac=1, random_state=42)
-    assert dss.x.evaluate().tolist() != x
-    assert list(sorted(dss.x.evaluate().tolist())) == x
+    ds_copy = ds.copy()
+
+    ds_string_filled = ds.fillna(value='NA')
+    assert ds_string_filled.obj.values[5] == 'NA'
+
+    ds_filled = ds.fillna(value=0)
+    assert ds_filled.obj.values[5] == 0
+
+    assert ds_filled.to_pandas_df(virtual=True).isna().any().any() == False
+    assert ds_filled.to_pandas_df(virtual=True).isna().any().any() == False
 
-    dss = ds.sample(n=1, random_state=42)
-    assert len(dss) == 1
+    ds_filled = ds.fillna(value=10, fill_masked=False)
+    assert ds_filled.n.values[6] == 10.
+    assert ds_filled.nm.values[6] == 10.
 
-    dss = ds.sample(n=100, random_state=42, replace=True)
-    assert len(dss) == 100
+    ds_filled = ds.fillna(value=-15, fill_nan=False)
+    assert ds_filled.m.values[7] == -15.
+    assert ds_filled.nm.values[7] == -15.
+    assert ds_filled.mi.values[7] == -15.
 
+    ds_filled = ds.fillna(value=-11, column_names=['nm', 'mi'])
+    assert ds_filled.to_pandas_df(virtual=True).isna().any().any() == True
+    assert ds_filled.to_pandas_df(column_names=['nm', 'mi']).isna().any().any() == False
 
-    dss = ds.sample(n=100, random_state=42, replace=True, weights='x')
-    assert 0 not in dss.x.evaluate().tolist()
-    assert 'bla' not in dss.x.evaluate().tolist()
+    state = ds_filled.state_get()
+    ds_copy.state_set(state)
+    np.testing.assert_array_equal(ds_copy['nm'].values, ds_filled['nm'].values)
+    np.testing.assert_array_equal(ds_copy['mi'].values, ds_filled['mi'].values)