Repository: alteryx/woodwork (python)

Issue:
Prevent Mutual Info from being calculated on unique columns
- If a column is unique, mutual info should not be calculated for that column.

Pull request:
Avoid calculating mutual information on unique columns
-  Closes #555 
- Had to update some of the existing tests to not be unique

Patch:
diff --git a/docs/source/release_notes.rst b/docs/source/release_notes.rst
index 315c751..f082873 100644
--- a/docs/source/release_notes.rst
+++ b/docs/source/release_notes.rst
@@ -6,13 +6,14 @@ Release Notes
     * Enhancements
     * Fixes
     * Changes
+        * Avoid calculating mutualinfo for non-unique columns (:pr:`563`)
     * Documentation Changes
     * Testing Changes
         * Update branch reference in tests workflow (:pr:`552`)
         * Fixed text on back arrow on install page (:pr:`564`)
 
     Thanks to the following people for contributing to this release:
-    :user:`gsheni`
+    :user:`gsheni`, :user:`tamargrey`
 
 **v0.0.9 February 5, 2021**
     * Enhancements
diff --git a/woodwork/datatable.py b/woodwork/datatable.py
index d1866ce..d9988c9 100644
--- a/woodwork/datatable.py
+++ b/woodwork/datatable.py
@@ -933,7 +933,7 @@ class DataTable(object):
             val_counts[col] = dt_list
         return val_counts
 
-    def _handle_nans_for_mutual_info(self, data):
+    def _replace_nans_for_mutual_info(self, data):
         """
         Remove NaN values in the dataframe so that mutual information can be calculated
 
@@ -941,12 +941,9 @@ class DataTable(object):
             data (pd.DataFrame): dataframe to use for calculating mutual information
 
         Returns:
-            pd.DataFrame: data with fully null columns removed and nans filled in
-                with either mean or mode
+            pd.DataFrame: data with nans replaced with either mean or mode
 
         """
-        # remove fully null columns
-        data = data.loc[:, data.columns[data.notnull().any()]]
 
         # replace or remove null values
         for column_name in data.columns[data.isnull().any()]:
@@ -1027,7 +1024,16 @@ class DataTable(object):
         if nrows is not None and nrows < data.shape[0]:
             data = data.sample(nrows)
 
-        data = self._handle_nans_for_mutual_info(data)
+        # remove fully null columns
+        not_null_cols = data.columns[data.notnull().any()]
+        if set(not_null_cols) != set(valid_columns):
+            data = data.loc[:, not_null_cols]
+        # remove columns that are unique
+        not_unique_cols = [col for col in data.columns if not data[col].is_unique]
+        if set(not_unique_cols) != set(valid_columns):
+            data = data.loc[:, not_unique_cols]
+
+        data = self._replace_nans_for_mutual_info(data)
         data = self._make_categorical_for_mutual_info(data, num_bins)
 
         # calculate mutual info for all pairs of columns


Test patch:
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 16c1576..ced01cb 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -77,7 +77,15 @@ jobs:
           ref = basename("${{ github.event.ref }}")
           is_dev = not any(pattern.match(ref) for pattern in patterns)
           print('::set-output name=is_dev::' + str(is_dev))
-
+      - if: ${{ steps.branch.outputs.is_dev == 'True' }}
+        name: Checkout repository
+        uses: actions/checkout@v2
+        with:
+          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
+      - if: ${{ steps.branch.outputs.is_dev == 'True' }}
+        name: Check if release notes were updated
+        run: cat docs/source/release_notes.rst | grep ":pr:\`${{ github.event.number }}\`"
   unit_tests:
     name: ${{ matrix.python_version }} unit tests ${{ matrix.optional_libraries }}
     runs-on: ubuntu-latest
diff --git a/woodwork/tests/conftest.py b/woodwork/tests/conftest.py
index a97f6e8..a05c3a9 100644
--- a/woodwork/tests/conftest.py
+++ b/woodwork/tests/conftest.py
@@ -303,11 +303,11 @@ def df_same_mi(request):
 @pytest.fixture()
 def df_mi_pandas():
     return pd.DataFrame({
-        'ints': pd.Series([1, 2, 3]),
+        'ints': pd.Series([1, 2, 1]),
         'bools': pd.Series([True, False, True]),
         'strs2': pd.Series(['bye', 'hi', 'bye']),
         'strs': pd.Series(['hi', 'hi', 'hi']),
-        'dates': pd.Series(['2020-01-01', '2019-01-02', '1997-01-04'])
+        'dates': pd.Series(['2020-01-01', '2020-01-01', '1997-01-04'])
     })
 
 
@@ -328,6 +328,33 @@ def df_mi(request):
     return request.getfixturevalue(request.param)
 
 
+@pytest.fixture()
+def df_mi_unique_pandas():
+    return pd.DataFrame({
+        'unique': pd.Series(['hi', 'bye', 'hello', 'goodbye']),
+        'unique_with_one_nan': pd.Series(['hi', 'bye', None, 'goodbye']),
+        'unique_with_nans': pd.Series([1, None, None, 2]),
+        'ints': pd.Series([1, 2, 1, 2]),
+    })
+
+
+@pytest.fixture()
+def df_mi_unique_dask(df_mi_unique_pandas):
+    dd = pytest.importorskip('dask.dataframe', reason='Dask not installed, skipping')
+    return dd.from_pandas(df_mi_unique_pandas, npartitions=1)
+
+
+@pytest.fixture()
+def df_mi_unique_koalas(df_mi_unique_pandas):
+    ks = pytest.importorskip('databricks.koalas', reason='Koalas not installed, skipping')
+    return ks.from_pandas(df_mi_unique_pandas)
+
+
+@pytest.fixture(params=['df_mi_unique_pandas', 'df_mi_unique_dask', 'df_mi_unique_koalas'])
+def df_mi_unique(request):
+    return request.getfixturevalue(request.param)
+
+
 @pytest.fixture()
 def categorical_df_pandas():
     return pd.DataFrame({
diff --git a/woodwork/tests/datatable/test_datatable.py b/woodwork/tests/datatable/test_datatable.py
index b343e43..82c78a7 100644
--- a/woodwork/tests/datatable/test_datatable.py
+++ b/woodwork/tests/datatable/test_datatable.py
@@ -2508,9 +2508,8 @@ def test_value_counts(categorical_df):
         assert len(val_cts_2[col]) == 2
 
 
-def test_datatable_handle_nans_for_mutual_info():
+def test_datatable_replace_nans_for_mutual_info():
     df_nans = pd.DataFrame({
-        'nans': pd.Series([None, None, None, None]),
         'ints': pd.Series([2, pd.NA, 5, 2], dtype='Int64'),
         'floats': pd.Series([3.3, None, 2.3, 1.3]),
         'bools': pd.Series([True, None, True, False]),
@@ -2520,11 +2519,10 @@ def test_datatable_handle_nans_for_mutual_info():
         'dates': pd.Series(['2020-01-01', None, '2020-01-02', '2020-01-03'])
     })
     dt_nans = DataTable(df_nans)
-    formatted_df = dt_nans._handle_nans_for_mutual_info(dt_nans.to_dataframe().copy())
+    formatted_df = dt_nans._replace_nans_for_mutual_info(dt_nans.to_dataframe().copy())
 
     assert isinstance(formatted_df, pd.DataFrame)
 
-    assert 'nans' not in formatted_df.columns
     assert formatted_df['ints'].equals(pd.Series([2, 3, 5, 2], dtype='Int64'))
     assert formatted_df['floats'].equals(pd.Series([3.3, 2.3, 2.3, 1.3], dtype='float'))
     assert formatted_df['bools'].equals(pd.Series([True, True, True, False], dtype='category'))
@@ -2572,26 +2570,25 @@ def test_datatable_mutual_information(df_mi):
     mi = dt.mutual_information()
     assert mi.shape[0] == 10
 
-    np.testing.assert_almost_equal(mi_between_cols('ints', 'bools', mi), 0.734, 3)
+    np.testing.assert_almost_equal(mi_between_cols('ints', 'bools', mi), 1.0, 3)
     np.testing.assert_almost_equal(mi_between_cols('ints', 'strs', mi), 0.0, 3)
     np.testing.assert_almost_equal(mi_between_cols('strs', 'bools', mi), 0, 3)
-    np.testing.assert_almost_equal(mi_between_cols('dates', 'ints', mi), 1.0, 3)
-    np.testing.assert_almost_equal(mi_between_cols('dates', 'bools', mi), 0.734, 3)
+    np.testing.assert_almost_equal(mi_between_cols('dates', 'ints', mi), 0.274, 3)
+    np.testing.assert_almost_equal(mi_between_cols('dates', 'bools', mi), 0.274, 3)
 
     mi_many_rows = dt.mutual_information(nrows=100000)
     pd.testing.assert_frame_equal(mi, mi_many_rows)
 
     mi = dt.mutual_information(nrows=1)
-    assert mi.shape[0] == 10
-    assert (mi['mutual_info'] == 1.0).all()
+    assert mi.shape[0] == 0
 
     mi = dt.mutual_information(num_bins=2)
     assert mi.shape[0] == 10
-    np.testing.assert_almost_equal(mi_between_cols('bools', 'ints', mi), .274, 3)
-    np.testing.assert_almost_equal(mi_between_cols('strs', 'ints', mi), 0, 3)
+    np.testing.assert_almost_equal(mi_between_cols('bools', 'ints', mi), 0, 3)
+    np.testing.assert_almost_equal(mi_between_cols('strs', 'ints', mi), 1.0, 3)
     np.testing.assert_almost_equal(mi_between_cols('bools', 'strs', mi), 0, 3)
-    np.testing.assert_almost_equal(mi_between_cols('dates', 'strs', mi), 0, 3)
-    np.testing.assert_almost_equal(mi_between_cols('dates', 'ints', mi), .274, 3)
+    np.testing.assert_almost_equal(mi_between_cols('dates', 'strs', mi), 1.0, 3)
+    np.testing.assert_almost_equal(mi_between_cols('dates', 'ints', mi), 1.0, 3)
 
     # Confirm that none of this changed the DataTable's underlying df
     pd.testing.assert_frame_equal(to_pandas(dt.to_dataframe()), to_pandas(original_df))
@@ -2617,6 +2614,17 @@ def test_mutual_info_sort(df_mi):
         assert mi['mutual_info'].iloc[i] >= mi['mutual_info'].iloc[i + 1]
 
 
+def test_mutual_info_unique(df_mi_unique):
+    dt = DataTable(df_mi_unique)
+    mi = dt.mutual_information()
+
+    cols_used = set(np.unique(mi[['column_1', 'column_2']].values))
+    assert 'unique' not in cols_used
+    assert 'unique_with_one_nan' not in cols_used
+    assert 'unique_with_nans' in cols_used
+    assert 'ints' in cols_used
+
+
 def test_mutual_info_dict(df_mi):
     dt = DataTable(df_mi)
     mi_dict = dt.mutual_information_dict()