Repository: modin-project/modin

Issue:
[BUG] drop_duplicates does not drop all duplicates
### System information
- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**: 18.04
- **Modin version** (`modin.__version__`): 0.8.0
- **Python version**: 3.6.9
- **Code we can use to reproduce**:

```python
import modin.pandas as pd
import pandas

pdf=pandas.DataFrame(
    [[5, 'ssssss0'], [0, 'ssssss4'], [3, 'ssssss5'], [3, 'ssssss5'], [7, 'ssssss6'], 
     [9, 'ssssss8'], [3, 'ssssss4'], [5, 'ssssss1'], [2, 'ssssss4'], [4, 'ssssss9'], 
     [7, 'ssssss8'], [6, 'ssssss1'], [8, 'ssssss1'], [8, 'ssssss7'], [1, 'ssssss9'], 
     [6, 'ssssss9'], [7, 'ssssss3'], [7, 'ssssss6'], [8, 'ssssss7'], [1, 'ssssss2'], 
     [5, 'ssssss0'], [9, 'ssssss3'], [8, 'ssssss5'], [9, 'ssssss9'], [4, 'ssssss4'], 
     [3, 'ssssss4'], [0, 'ssssss6'], [3, 'ssssss4'], [5, 'ssssss4'], [0, 'ssssss3'], 
     [2, 'ssssss4'], [3, 'ssssss4'], [8, 'ssssss8'], [1, 'ssssss4'], [3, 'ssssss3'], 
     [3, 'ssssss7'], [3, 'ssssss5'], [7, 'ssssss5'], [0, 'ssssss0'], [1, 'ssssss1'], 
     [9, 'ssssss5'], [9, 'ssssss9'], [0, 'ssssss3'], [4, 'ssssss0'], [7, 'ssssss5'], 
     [3, 'ssssss0'], [2, 'ssssss1'], [7, 'ssssss2'], [2, 'ssssss4'], [0, 'ssssss2']], columns=['a', 'b'])

mdf = pd.DataFrame(pdf)

print(pdf.drop_duplicates().shape)
print(mdf.drop_duplicates().shape)
```
<!--
You can obtain the Modin version with

python -c "import modin; print(modin.__version__)"
-->

### Describe the problem
<!-- Describe the problem clearly here. -->
Modin does not drop all duplicates (with ray backend)

### Source code / logs
<!-- Include any logs or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached. Try to provide a reproducible test case that is the bare minimum necessary to generate the problem. -->

```
(37, 2)
(41, 2)
```


Discussion:
And here is another issue of drop_duplicates, it sometimes drops rows with all nulls, but pandas doesn't, so the behavior is not consistent

```python
import modin.pandas as pd
import pandas

nan=float("nan")
pdf=pandas.DataFrame(
    [[5.0, 4.0], [0.0, nan], [nan, 8.0], [3.0, 1.0], [7.0, 1.0], 
     [nan, 7.0], [nan, nan], [5.0, nan], [nan, nan], [4.0, 6.0], 
     [7.0, nan], [nan, nan], [nan, nan], [nan, nan], [1.0, nan], 
     [nan, nan], [nan, 4.0], [7.0, 4.0], [8.0, 6.0], [1.0, nan], 
     [5.0, 4.0], [nan, nan], [nan, nan], [nan, nan], [4.0, 8.0], 
     [3.0, 4.0], [nan, nan], [3.0, 7.0], [nan, 5.0], [nan, 5.0]], columns=['a', 'b'])

mdf = pd.DataFrame(pdf)

print(pdf.drop_duplicates().sort_values(["a","b"]))
print(mdf.drop_duplicates().sort_values(["a","b"])) # this does not have (nan,nan) row
```

The drop null behavior itself is not consistent inside modin drop_duplicates

the following works fine:

```python
import modin.pandas as pd
import pandas

nan=float("nan")
pdf=pandas.DataFrame([[5.0, 4.0], [nan, nan]], columns=['a', 'b'])

mdf = pd.DataFrame(pdf)

print(pdf.drop_duplicates().sort_values(["a","b"]))
print(mdf.drop_duplicates().sort_values(["a","b"]))
```
Thanks @goodwanghan for the report! I can reproduce the mismatch.

This is implemented with a deterministic hashing function. In this case, for some reason the hash function is not producing identical hashes on some of the common rows, this will be good to investigate.
Root cause is in #1987. Looks like functions work in each partition separately without any reduction.

Resolving patch:
diff --git a/modin/pandas/dataframe.py b/modin/pandas/dataframe.py
--- a/modin/pandas/dataframe.py
+++ b/modin/pandas/dataframe.py
@@ -282,11 +282,15 @@ def duplicated(self, subset=None, keep="first"):
         Returns:
             Series
         """
+        import hashlib
+
         df = self[subset] if subset is not None else self
         # if the number of columns we are checking for duplicates is larger than 1, we must
         # hash them to generate a single value that can be compared across rows.
         if len(df.columns) > 1:
-            hashed = df.apply(lambda s: hash(tuple(s)), axis=1).to_frame()
+            hashed = df.apply(
+                lambda s: hashlib.new("md5", str(tuple(s)).encode()).hexdigest(), axis=1
+            ).to_frame()
         else:
             hashed = df
         duplicates = hashed.apply(lambda s: s.duplicated(keep=keep)).squeeze(axis=1)


Test patch:
diff --git a/modin/pandas/test/dataframe/test_window.py b/modin/pandas/test/dataframe/test_window.py
--- a/modin/pandas/test/dataframe/test_window.py
+++ b/modin/pandas/test/dataframe/test_window.py
@@ -24,6 +24,8 @@
     name_contains,
     test_data_values,
     test_data_keys,
+    test_data_with_duplicates_values,
+    test_data_with_duplicates_keys,
     no_numeric_dfs,
     quantiles_keys,
     quantiles_values,
@@ -211,7 +213,9 @@ def test_diff(request, data, axis, periods):
         df_equals(modin_result, pandas_result)
 
 
-@pytest.mark.parametrize("data", test_data_values, ids=test_data_keys)
+@pytest.mark.parametrize(
+    "data", test_data_with_duplicates_values, ids=test_data_with_duplicates_keys
+)
 @pytest.mark.parametrize(
     "keep", ["last", "first", False], ids=["last", "first", "False"]
 )
diff --git a/modin/pandas/test/test_series.py b/modin/pandas/test/test_series.py
--- a/modin/pandas/test/test_series.py
+++ b/modin/pandas/test/test_series.py
@@ -31,6 +31,8 @@
     test_data,
     test_data_values,
     test_data_keys,
+    test_data_with_duplicates_values,
+    test_data_with_duplicates_keys,
     test_string_data_values,
     test_string_data_keys,
     test_string_list_data_values,
@@ -1391,7 +1393,9 @@ def test_drop():
         modin_series.drop(None, None, None, None)
 
 
-@pytest.mark.parametrize("data", test_data_values, ids=test_data_keys)
+@pytest.mark.parametrize(
+    "data", test_data_with_duplicates_values, ids=test_data_with_duplicates_keys
+)
 @pytest.mark.parametrize(
     "keep", ["last", "first", False], ids=["last", "first", "False"]
 )
@@ -1527,7 +1531,9 @@ def test_dt():
     df_equals(modin_series.dt.to_timestamp(), pandas_series.dt.to_timestamp())
 
 
-@pytest.mark.parametrize("data", test_data_values, ids=test_data_keys)
+@pytest.mark.parametrize(
+    "data", test_data_with_duplicates_values, ids=test_data_with_duplicates_keys
+)
 @pytest.mark.parametrize(
     "keep", ["last", "first", False], ids=["last", "first", "False"]
 )
diff --git a/modin/pandas/test/utils.py b/modin/pandas/test/utils.py
--- a/modin/pandas/test/utils.py
+++ b/modin/pandas/test/utils.py
@@ -101,6 +101,11 @@
     "col{}".format(int(NCOLS / 2))
 )
 
+for col in test_data["float_nan_data"]:
+    for row in range(NROWS // 2):
+        if row % 16 == 0:
+            test_data["float_nan_data"][col][row] = np.NaN
+
 test_data_values = list(test_data.values())
 test_data_keys = list(test_data.keys())
 
@@ -128,21 +133,22 @@
         ]
         for i in range(NCOLS)
     },
-    "subset_duplicates": {
-        "col{}".format(i): [
-            i if j % 7 == 0 and i in [1, 3, 7] else x
-            for j, x in enumerate(range(NROWS))
-        ]
-        for i in range(NCOLS)
-    },
     "has_name_column": {
         "name": ["one", "two", "two", "three"],
         "col1": [1, 2, 2, 3],
         "col3": [10, 20, 20, 3],
         "col7": [100, 201, 200, 300],
     },
+    "str_columns": {
+        "col_str{}".format(int((i - NCOLS / 2) % NCOLS + 1)): [
+            "s" + str(x % 5) for x in range(NROWS)
+        ]
+        for i in range(NCOLS)
+    },
 }
 
+test_data_with_duplicates["float_nan"] = test_data["float_nan_data"]
+
 test_data_small = {
     "small": {
         "col0": [1, 2, 3, 4],