Repository: deepchecks/deepchecks

Issue:
Exception when running DataDuplicates check
To reproduce:
https://www.kaggle.com/itay94/notebookf8c78e84d7

```
---------------------------------------------------------------------------
MemoryError                               Traceback (most recent call last)
/tmp/ipykernel_34/3771541214.py in <module>
      1 from deepchecks.checks import DataDuplicates
      2 
----> 3 DataDuplicates().run(ds_train)

/opt/conda/lib/python3.7/site-packages/deepchecks/base/check.py in wrapped(*args, **kwargs)
    275     @wraps(func)
    276     def wrapped(*args, **kwargs):
--> 277         result = func(*args, **kwargs)
    278         if not isinstance(result, CheckResult):
    279             raise DeepchecksValueError(f'Check {class_instance.name()} expected to return CheckResult bot got: '

/opt/conda/lib/python3.7/site-packages/deepchecks/checks/integrity/data_duplicates.py in run(self, dataset, model)
     71             raise DeepchecksValueError('Dataset does not contain any data')
     72 
---> 73         group_unique_data = df[data_columns].groupby(data_columns, dropna=False).size()
     74         n_unique = len(group_unique_data)
     75 

/opt/conda/lib/python3.7/site-packages/pandas/core/groupby/groupby.py in size(self)
   1834             result = result.rename("size").reset_index()
   1835 
-> 1836         return self._reindex_output(result, fill_value=0)
   1837 
   1838     @final

/opt/conda/lib/python3.7/site-packages/pandas/core/groupby/groupby.py in _reindex_output(self, output, fill_value)
   3163         levels_list = [ping.group_index for ping in groupings]
   3164         index, _ = MultiIndex.from_product(
-> 3165             levels_list, names=self.grouper.names
   3166         ).sortlevel()
   3167 

/opt/conda/lib/python3.7/site-packages/pandas/core/indexes/multi.py in from_product(cls, iterables, sortorder, names)
    618 
    619         # codes are all ndarrays, so cartesian_product is lossless
--> 620         codes = cartesian_product(codes)
    621         return cls(levels, codes, sortorder=sortorder, names=names)
    622 

/opt/conda/lib/python3.7/site-packages/pandas/core/reshape/util.py in cartesian_product(X)
     52         b = np.zeros_like(cumprodX)
     53 
---> 54     return [tile_compat(np.repeat(x, b[i]), np.product(a[i])) for i, x in enumerate(X)]
     55 
     56 

/opt/conda/lib/python3.7/site-packages/pandas/core/reshape/util.py in <listcomp>(.0)
     52         b = np.zeros_like(cumprodX)
     53 
---> 54     return [tile_compat(np.repeat(x, b[i]), np.product(a[i])) for i, x in enumerate(X)]
     55 
     56 

<__array_function__ internals> in repeat(*args, **kwargs)

/opt/conda/lib/python3.7/site-packages/numpy/core/fromnumeric.py in repeat(a, repeats, axis)
    477 
    478     """
--> 479     return _wrapfunc(a, 'repeat', repeats, axis=axis)
    480 
    481 

/opt/conda/lib/python3.7/site-packages/numpy/core/fromnumeric.py in _wrapfunc(obj, method, *args, **kwds)
     56 
     57     try:
---> 58         return bound(*args, **kwds)
     59     except TypeError:
     60         # A TypeError occurs if the object does have such a method in its

MemoryError: Unable to allocate 589. PiB for an array with shape (331491645779374080,) and data type int16
```


Discussion:
Pandas has a bug with categorical features with many unique values. Please see:

pandas-dev/pandas#45128

As a workaround, I suggest casting to object dtype columns with many categories

Resolving patch:
diff --git a/deepchecks/checks/integrity/data_duplicates.py b/deepchecks/checks/integrity/data_duplicates.py
--- a/deepchecks/checks/integrity/data_duplicates.py
+++ b/deepchecks/checks/integrity/data_duplicates.py
@@ -70,6 +70,11 @@ def run(self, dataset: Dataset, model=None) -> CheckResult:
         if n_samples == 0:
             raise DeepchecksValueError('Dataset does not contain any data')
 
+        # HACK: pandas have bug with groupby on category dtypes, so until it fixed, change dtypes manually
+        category_columns = df.dtypes[df.dtypes == 'category'].index.tolist()
+        if category_columns:
+            df = df.astype({c: 'object' for c in category_columns})
+
         group_unique_data = df[data_columns].groupby(data_columns, dropna=False).size()
         n_unique = len(group_unique_data)
 
diff --git a/deepchecks/checks/integrity/label_ambiguity.py b/deepchecks/checks/integrity/label_ambiguity.py
--- a/deepchecks/checks/integrity/label_ambiguity.py
+++ b/deepchecks/checks/integrity/label_ambiguity.py
@@ -69,7 +69,13 @@ def run(self, dataset: Dataset, model=None) -> CheckResult:
 
         label_col = dataset.label_name
 
-        group_unique_data = dataset.data.groupby(dataset.features, dropna=False)
+        # HACK: pandas have bug with groupby on category dtypes, so until it fixed, change dtypes manually
+        df = dataset.data
+        category_columns = df.dtypes[df.dtypes == 'category'].index.tolist()
+        if category_columns:
+            df = df.astype({c: 'object' for c in category_columns})
+
+        group_unique_data = df.groupby(dataset.features, dropna=False)
         group_unique_labels = group_unique_data.nunique()[label_col]
 
         num_ambiguous = 0


Test patch:
diff --git a/tests/checks/integrity/data_duplicate_test.py b/tests/checks/integrity/data_duplicate_test.py
--- a/tests/checks/integrity/data_duplicate_test.py
+++ b/tests/checks/integrity/data_duplicate_test.py
@@ -28,6 +28,20 @@ def test_data_duplicates():
     assert_that(check_obj.run(duplicate_data).value, close_to(0.40, 0.01))
 
 
+def test_data_duplicates_categorical_dtypes():
+    """We used to have a bug when using groupby on category dtypes, this test ensures it doesn't return """
+    data = {
+        'a': np.random.randint(0, 1000, 300000),
+        'b': np.random.randint(0, 1000, 300000),
+        'c': np.random.randint(0, 1000, 300000),
+        'd': np.random.randint(0, 1000, 300000),
+        'e': np.random.randint(0, 1000, 300000)
+    }
+    duplicate_data = pd.DataFrame(data).astype('category')
+    check_obj = DataDuplicates()
+    assert_that(check_obj.run(duplicate_data).value, close_to(0.0, 0.001))
+
+
 def test_data_duplicates_columns():
     duplicate_data = pd.DataFrame({'col1': [1, 2, 1, 2, 1, 2, 1, 2, 1, 2],
                                    'col2': [1, 2, 1, 2, 1, 2, 1, 2, 1, 2],
diff --git a/tests/checks/integrity/label_ambiguity_test.py b/tests/checks/integrity/label_ambiguity_test.py
--- a/tests/checks/integrity/label_ambiguity_test.py
+++ b/tests/checks/integrity/label_ambiguity_test.py
@@ -26,7 +26,9 @@ def test_label_ambiguity():
         'col3': [1, 1, 1, 2, 2, 2]*100,
         'label': [1, 1, 2, 2, 3, 4]*100
     }
-    ds = Dataset(pd.DataFrame(data), label_name='label')
+    dataframe = pd.DataFrame(data)
+    dataframe = dataframe.astype({'col1': 'category'})
+    ds = Dataset(dataframe, label_name='label')
     check = LabelAmbiguity()
     # Act
     result = check.run(ds)