Repository: vaexio/vaex

Issue:
LabelEncoder new columns are object, should be int.
After running LabelEncoder, we get new columns of type object, although all values are ints.
This creates a new level of challenge with missing values, as you get None values in a column with ints. 


Discussion:
```
import vaex
from vaex.ml.transformations import LabelEncoder

df = vaex.from_dict({'a': [1, None, 1], 'b': ['a', None, None], 'c': [None, None, None]})
le = LabelEncoder(features=['a', 'b', 'c'], allow_unseen=True)
df = le.fit_transform(df)

>>> df.dtypes
a                         object
b                  <class 'str'>
c                        float64
label_encoded_a            int64
label_encoded_b           object <<<< this is the problem
label_encoded_c            int64
```

A good example of why it is a problem:
```
>>> df['label_encoded_b'].astype('int')
...
TypeError: int() argument must be a string, a bytes-like object or a number, not 'NoneType'
```
Workaround:
```
df['label_encoded_b'].fillna('0').astype('int')
```

Resolving patch:
diff --git a/packages/vaex-core/vaex/array_types.py b/packages/vaex-core/vaex/array_types.py
--- a/packages/vaex-core/vaex/array_types.py
+++ b/packages/vaex-core/vaex/array_types.py
@@ -33,6 +33,13 @@ def same_type(type1, type2):
         return False
 
 
+def tolist(ar):
+    if isinstance(ar, supported_arrow_array_types):
+        return ar.to_pylist()
+    else:
+        return ar.tolist()
+
+
 def to_numpy(x, strict=False):
     import vaex.arrow.convert
     import vaex.column
diff --git a/packages/vaex-core/vaex/dataframe.py b/packages/vaex-core/vaex/dataframe.py
--- a/packages/vaex-core/vaex/dataframe.py
+++ b/packages/vaex-core/vaex/dataframe.py
@@ -509,10 +509,19 @@ def reduce(a, b):
         if not dropnan:
             if ordered_set.has_nan:
                 keys = [np.nan] + keys
-        if not dropmissing:
-            if ordered_set.has_null:
-                keys = [np.ma.core.MaskedConstant()] + keys
-        keys = np.asarray(keys)
+        if self.is_string(expression):
+            if not dropmissing:
+                if ordered_set.has_null:
+                    # arrow handles None as missing
+                    keys = [None] + keys
+            keys = pa.array(keys)
+        else:
+            masked = False
+            if not dropmissing:
+                if ordered_set.has_null:
+                    masked = True
+                    keys = [np.ma.core.MaskedConstant()] + keys
+            keys = np.ma.asarray(keys) if masked else np.asarray(keys)
         if return_inverse:
             return keys, inverse
         else:
@@ -5000,6 +5009,10 @@ def ordinal_encode(self, column, values=None, inplace=False):
         # codes point to the index of found_values
         # meaning: found_values[codes[0]] == ds[column].values[0]
         found_values, codes = df_unfiltered.unique(column, return_inverse=True)
+        if isinstance(found_values, array_types.supported_arrow_array_types):
+            # elements of arrow arrays are not in arrow arrays, e.g. ar[0] in ar is False
+            # see tests/arrow/assumptions_test.py::test_in_pylist
+            found_values = found_values.to_pylist()
         if values is None:
             values = found_values
         else:
diff --git a/packages/vaex-ml/vaex/ml/transformations.py b/packages/vaex-ml/vaex/ml/transformations.py
--- a/packages/vaex-ml/vaex/ml/transformations.py
+++ b/packages/vaex-ml/vaex/ml/transformations.py
@@ -160,7 +160,7 @@ def fit(self, df):
         '''
 
         for feature in self.features:
-            labels = df[feature].unique().tolist()
+            labels = vaex.array_types.tolist(df[feature].unique())
             self.labels_[feature] = dict(zip(labels, np.arange(len(labels))))
 
     def transform(self, df):
@@ -227,9 +227,15 @@ def fit(self, df):
         uniques = []
         for i in self.features:
             expression = _ensure_strings_from_expressions(i)
-            unique = df.unique(expression)
-            unique = np.sort(unique)  # this can/should be optimized with @delay
-            uniques.append(unique.tolist())
+            unique_values = vaex.array_types.tolist(df.unique(expression))
+
+            if None in unique_values:
+                unique_values.remove(None)
+                unique_values.sort()
+                unique_values.insert(0, None)  # This is done in place
+            else:
+                unique_values.sort()
+            uniques.append(unique_values)
         self.uniques_ = uniques
 
     def transform(self, df):
@@ -243,9 +249,14 @@ def transform(self, df):
         # for each feature, add a virtual column for each unique entry
         for i, feature in enumerate(self.features):
             for j, value in enumerate(self.uniques_[i]):
-                column_name = self.prefix + feature + '_' + str(value)
-                copy.add_virtual_column(column_name, 'where({feature} == {value}, {one}, {zero})'.format(
-                                        feature=feature, value=repr(value), one=self.one, zero=self.zero))
+                str_value = str(value) if value is not None else 'missing'
+                column_name = self.prefix + feature + '_' + str_value
+                if value is None:
+                    copy[column_name] = copy.func.where(copy[feature].ismissing(), self.one, self.zero)
+                elif isinstance(value, np.float) and np.isnan(value):
+                    copy[column_name] = copy.func.where(copy[feature].isnan(), self.one, self.zero)
+                else:
+                    copy[column_name] = copy.func.where(copy[feature] == value, self.one, self.zero)
         return copy
 
 


Test patch:
diff --git a/tests/arrow/assumptions_test.py b/tests/arrow/assumptions_test.py
--- a/tests/arrow/assumptions_test.py
+++ b/tests/arrow/assumptions_test.py
@@ -12,6 +12,16 @@ def test_non_native():
     with pytest.raises(pa.lib.ArrowNotImplementedError):
         pa.array(x)
 
+@pytest.mark.skip(reason="Only the case for Arrow 0.17, leaving this is as 'documentation'")
+def test_null_behaviour():
+    assert pa.NULL in ['Confused'], "Arrow 0.17 "
+
+
+def test_in_pylist():
+    # this reflects the to_pylist in DataFrame.ordinal_encode
+    ar = pa.array(['red', 'green'])
+    assert ar[0] not in ar.to_pylist(), "Arrow 1.0.1 says no"
+
 
 def test_cannot_convert_nulls_to_masked():
     x = np.arange(4)
diff --git a/tests/ml/ml_test.py b/tests/ml/ml_test.py
--- a/tests/ml/ml_test.py
+++ b/tests/ml/ml_test.py
@@ -216,6 +216,38 @@ def test_one_hot_encoding():
     ohe = vaex.ml.OneHotEncoder(features=['kids', 'animals', 'numbers'])
     ohe.fit_transform(ds)
 
+def test_one_hot_encoding_with_na():
+    x = ['Reggie', 'Michael', None, 'Reggie']
+    y = [31, 23, np.nan, 31]
+    df_train = vaex.from_arrays(x=x, y=y)
+
+    x = ['Michael', 'Reggie', None, None]
+    y = [23, 31, np.nan, np.nan]
+    df_test = vaex.from_arrays(x=x, y=y)
+
+
+    enc = vaex.ml.OneHotEncoder(features=['x', 'y'])
+    enc.fit(df_train)
+
+    assert enc.uniques_[0] == [None, 'Michael', 'Reggie']
+    np.testing.assert_array_equal(enc.uniques_[1], [np.nan, 23.0, 31.0])
+
+    df_train = enc.transform(df_train)
+    assert df_train.x_missing.tolist() == [0, 0, 1, 0]
+    assert df_train.x_Michael.tolist() == [0, 1, 0, 0]
+    assert df_train.x_Reggie.tolist() == [1, 0, 0, 1]
+    assert df_train['y_23.0'].tolist() == [0, 1, 0, 0]
+    assert df_train['y_31.0'].tolist() == [1, 0, 0, 1]
+    assert df_train['y_nan'].tolist() == [0, 0, 1, 0]
+
+    df_test = enc.transform(df_test)
+    assert df_test.x_missing.tolist() == [0, 0, 1, 1]
+    assert df_test.x_Michael.tolist() == [1, 0, 0, 0]
+    assert df_test.x_Reggie.tolist() == [0, 1, 0, 0]
+    assert df_test['y_23.0'].tolist() == [1, 0, 0, 0]
+    assert df_test['y_31.0'].tolist() == [0, 1, 0, 0]
+    assert df_test['y_nan'].tolist() == [0, 0, 1, 1]
+
 
 def test_maxabs_scaler():
     x = np.array([-2.65395789, -7.97116295, -4.76729177, -0.76885033, -6.45609635])
diff --git a/tests/unique_test.py b/tests/unique_test.py
--- a/tests/unique_test.py
+++ b/tests/unique_test.py
@@ -1,25 +1,31 @@
-from common import *
+from common import small_buffer
+
+import numpy as np
+
+import vaex
+
 
 def test_unique_arrow():
     ds = vaex.from_arrays(x=vaex.string_column(['a', 'b', 'a', 'a', 'a', 'b', 'b', 'b', 'b', 'a']))
     with small_buffer(ds, 2):
-        assert set(ds.unique(ds.x).astype('U').tolist()) == {'a', 'b'}
+        assert set(ds.unique(ds.x).to_pylist()) == {'a', 'b'}
         values, index = ds.unique(ds.x, return_inverse=True)
-        assert values[index].tolist() == ds.x.tolist()
+        assert np.array(values)[index].tolist() == ds.x.tolist()
 
 
 def test_unique():
     ds = vaex.from_arrays(colors=['red', 'green', 'blue', 'green'])
     with small_buffer(ds, 2):
-        assert set(ds.unique(ds.colors).astype('U').tolist()) == {'red', 'green', 'blue'}
+        assert set(ds.unique(ds.colors).to_pylist()) == {'red', 'green', 'blue'}
         values, index = ds.unique(ds.colors, return_inverse=True)
-        assert values[index].tolist() == ds.colors.tolist()
+        assert np.array(values)[index].tolist() == ds.colors.tolist()
 
     ds = vaex.from_arrays(x=['a', 'b', 'a', 'a', 'a', 'b', 'b', 'b', 'b', 'a'])
     with small_buffer(ds, 2):
-        assert set(ds.unique(ds.x).astype('U').tolist()) == {'a', 'b'}
+        assert set(ds.unique(ds.x).to_pylist()) == {'a', 'b'}
         values, index = ds.unique(ds.x, return_inverse=True)
-        assert values[index].tolist() == ds.x.tolist()
+        assert np.array(values)[index].tolist() == ds.x.tolist()
+
 
 def test_unique_f4():
     x = np.array([np.nan, 0, 1, np.nan, 2, np.nan], dtype='f4')
@@ -38,9 +44,21 @@ def test_unique_nan():
         assert values[~mask].tolist() == df.x.values[~mask].tolist()
         # assert indices.tolist() == [0, 1, 2, 0, 3, 0]
 
+
 def test_unique_missing():
     # Create test databn
-    x = np.array([None, 'A', 'B', -1, 0, 2, '', '', None, None, None, np.nan, np.nan, np.nan, np.nan,])
+    x = np.array([None, 'A', 'B', -1, 0, 2, '', '', None, None, None, np.nan, np.nan, np.nan, np.nan])
     df = vaex.from_arrays(x=x)
     uniques = df.x.unique(dropnan=True).tolist()
     assert set(uniques) == set(['', 'A', 'B', -1, 0, 2, None])
+
+
+def test_unique_string_missing():
+    x = ['John', None, 'Sally', None, '0.0']
+    df = vaex.from_arrays(x=x)
+    result = df.x.unique().to_pylist()
+
+    assert len(result) == 4
+    assert'John' in result
+    assert None in result
+    assert 'Sally'