Repository: vaexio/vaex

Issue:
[BUG-REPORT]
Thank you for reaching out and helping us improve Vaex!

Before you submit a new Issue, please read through the [documentation](https://docs.vaex.io/en/latest/). Also, make sure you search through the Open and Closed Issues - your problem may already be discussed or addressed.

**Description**
Please provide a clear and concise description of the problem. This should contain all the steps needed to reproduce the problem. A minimal code example that exposes the problem is very appreciated.

Following code returns an incorrect result

```python
df=vaex.from_arrays(x=np.array([1.01,1.02,1.03]))

df.x.isin([1])
```

Expression = isin_set(x, var_isin_ordered_set)
Length: 3 dtype: bool (expression)
0 True
1 True
2 True

should be all false

**Software information**
 - Vaex version (`import vaex; vaex.__version__)`: 4.1
 - Vaex was installed via: pip / conda-forge / from source  conda-forge
 - OS: fedora 33

**Additional information**
Please state any supplementary information or provide additional context for the problem (e.g. screenshots, data, etc..).



Discussion:
I was able to reproduce the issue and hopefully will have a PR soon.

Resolving patch:
diff --git a/packages/vaex-core/vaex/expression.py b/packages/vaex-core/vaex/expression.py
--- a/packages/vaex-core/vaex/expression.py
+++ b/packages/vaex-core/vaex/expression.py
@@ -864,6 +864,7 @@ def isin(self, values, use_hashmap=True):
         """
         if use_hashmap:
             # easiest way to create a set is using the vaex dataframe
+            values = np.array(values, dtype=self.dtype.numpy)  # ensure that values are the same dtype as the expression (otherwise the set downcasts at the C++ level during execution)
             df_values = vaex.from_arrays(x=values)
             ordered_set = df_values._set(df_values.x)
             var = self.df.add_variable('var_isin_ordered_set', ordered_set, unique=True)
@@ -872,7 +873,7 @@ def isin(self, values, use_hashmap=True):
             if self.is_string():
                 values = vaex.column._to_string_sequence(values)
             else:
-                values = np.array(values, dtype=self.dtype)
+                values = np.array(values, dtype=self.dtype.numpy)
             var = self.df.add_variable('isin_values', values, unique=True)
             return self.df['isin(%s, %s)' % (self, var)]
 
diff --git a/packages/vaex-core/vaex/functions.py b/packages/vaex-core/vaex/functions.py
--- a/packages/vaex-core/vaex/functions.py
+++ b/packages/vaex-core/vaex/functions.py
@@ -2485,10 +2485,17 @@ def _isin(x, values):
         # but numpy doesn't know what to do with that
         if hasattr(values, 'to_numpy'):
             values = values.to_numpy()
-        if np.ma.isMaskedArray(x):
-            return np.ma.isin(x, values)
+        mask = isnan(values)
+        if np.any(mask):
+            if np.ma.isMaskedArray(x):
+                return np.ma.isin(x, values) | isnan(x)
+            else:
+                return np.isin(x, values) | isnan(x)
         else:
-            return np.isin(x, values)
+            if np.ma.isMaskedArray(x):
+                return np.ma.isin(x, values)
+            else:
+                return np.isin(x, values)
 
 
 @register_function(name='isin_set', on_expression=False)


Test patch:
diff --git a/tests/isin_test.py b/tests/isin_test.py
--- a/tests/isin_test.py
+++ b/tests/isin_test.py
@@ -1,8 +1,11 @@
+import pytest
+
 import numpy as np
 import vaex
 
 
-def test_isin():
+@pytest.mark.parametrize("use_hashmap", [False, True])
+def test_isin(use_hashmap):
     x = np.array([1.01, 2.02, 3.03])
     y = np.array([1, 3, 5])
     s = np.array(['dog', 'cat', 'mouse'])
@@ -12,15 +15,13 @@ def test_isin():
     n = np.array([-5, np.nan, 1])
     df = vaex.from_arrays(x=x, y=y, s=s, sm=sm, w=w, m=m, n=n)
 
-    assert df.x.isin([1, 2.02, 5, 6]).tolist() == [False, True, False]
-    assert df.y.isin([5, -1, 0]).tolist() == [False, False, True]
-    assert df.s.isin(['elephant', 'dog']).tolist() == [True, False, False]
-    assert df.sm.isin(['cat', 'dog']).tolist() == [True, True, False]
-    assert df.w.isin([2, None]).tolist() == [True, False, True]
-    assert df.m.isin([1, 2, 3]).tolist() == [False, False, True]
-    assert df.n.isin([2, np.nan]).tolist() == [False, True, False]
-    # TODO: this fails, we should have a separate issue for it
-    assert df.n.isin([2, np.nan]).tolist() == [False, True, False]
+    assert df.x.isin([1, 2.02, 5, 6], use_hashmap=use_hashmap).tolist() == [False, True, False]
+    assert df.y.isin([5, -1, 0], use_hashmap=use_hashmap).tolist() == [False, False, True]
+    assert df.s.isin(['elephant', 'dog'], use_hashmap=use_hashmap).tolist() == [True, False, False]
+    assert df.sm.isin(['cat', 'dog'], use_hashmap=use_hashmap).tolist() == [True, True, False]
+    assert df.w.isin([2, None], use_hashmap=use_hashmap).tolist() == [True, False, True]
+    assert df.m.isin([1, 2, 3], use_hashmap=use_hashmap).tolist() == [False, False, True]
+    assert df.n.isin([2, np.nan], use_hashmap=use_hashmap).tolist() == [False, True, False]
 
 
 def test_isin_object():
@@ -32,3 +33,14 @@ def test_isin_object():
 
     assert expr_x.tolist() == [True, False, False]
     assert expr_y.tolist() == [False, True, False]
+
+
+@pytest.mark.parametrize("use_hashmap", [False, True])
+def test_isin_diff_dtypes(use_hashmap):
+    x = np.array([1.01, 2.02, 3.03])
+    s = np.array(['dog', 'cat', 'mouse'])
+    sm = np.array(['dog', 'cat', None])
+    df = vaex.from_arrays(x=x, s=s, sm=sm)
+
+    assert df.x.isin([1], use_hashmap=use_hashmap).tolist() == [False, False, False]
+    assert df.x.isin([1.01], use_hashmap=use_hashmap).tolist() == [True, False, False]