Repository: alteryx/woodwork (python)

Issue:
Use Unknown logical type for null columns
Based off discussion in #1038, it makes more sense to infer columns that are entirely null (i.e. made entirely up of `None`, `np.nan`, or `pd.NA`) as `Unknown` logical type instead of `Double` or `Categorical`. This way, entirely null columns are all handled in a consistent manner.

Pull request description:
Infer logical type of null columns as Unknown
- Columns that are entirely null (made up of `None`, `np.nan`, or `pd.NA`) are inferred as `Unknown` instead of `Double`/`Categorical`.
- Closes #1044 

Patch:
diff --git a/docs/source/release_notes.rst b/docs/source/release_notes.rst
index e7c3894..9a3e363 100644
--- a/docs/source/release_notes.rst
+++ b/docs/source/release_notes.rst
@@ -2,15 +2,17 @@
 
 Release Notes
 -------------
-.. Future Release
-    ==============
+Future Release
+==============
     * Enhancements
     * Fixes
     * Changes
+        * Entirely null columns are now inferred as the Unknown logical type (:pr:`1043`)
     * Documentation Changes
     * Testing Changes
 
-.. Thanks to the following people for contributing to this release:
+    Thanks to the following people for contributing to this release:
+    :user:`frances-h`
 
 v0.5.0 Jul 7, 2021
 ==================
diff --git a/woodwork/type_sys/type_system.py b/woodwork/type_sys/type_system.py
index 4944ff5..8999ec8 100644
--- a/woodwork/type_sys/type_system.py
+++ b/woodwork/type_sys/type_system.py
@@ -261,6 +261,10 @@ class TypeSystem(object):
         if ks and isinstance(series, ks.Series):
             series = series.head(100000).to_pandas()
 
+        # Special case: if the entire column is Null or NaN, use the Unknown logical type
+        if all(series.isnull()):
+            return Unknown()
+
         def get_inference_matches(types_to_check, series, type_matches=[]):
             # Since NaturalLanguage isn't inferred by default, make sure to check
             # any children of NaturalLanguage, otherwise they never get evaluated


Test patch:
diff --git a/woodwork/tests/accessor/test_statistics.py b/woodwork/tests/accessor/test_statistics.py
index cc2020b..ca113b5 100644
--- a/woodwork/tests/accessor/test_statistics.py
+++ b/woodwork/tests/accessor/test_statistics.py
@@ -118,7 +118,7 @@ def test_accessor_make_categorical_for_mutual_info():
 
 
 def test_mutual_info_same(df_same_mi):
-    df_same_mi.ww.init()
+    df_same_mi.ww.init(logical_types={'nans': Categorical()})
 
     mi = df_same_mi.ww.mutual_information()
 
diff --git a/woodwork/tests/type_system/conftest.py b/woodwork/tests/type_system/conftest.py
index 40af2b0..72d1100 100644
--- a/woodwork/tests/type_system/conftest.py
+++ b/woodwork/tests/type_system/conftest.py
@@ -104,8 +104,8 @@ def bools(request):
 @pytest.fixture
 def pandas_datetimes():
     return [
-        pd.Series(['3/11/2000', '3/12/2000', '3/13/2000', '3/14/2000']),
-        pd.Series(['3/11/2000', np.nan, '3/13/2000', '3/14/2000']),
+        pd.Series(['2000-3-11', '2000-3-12', '2000-03-13', '2000-03-14']),
+        pd.Series(['2000-3-11', np.nan, '2000-03-13', '2000-03-14']),
     ]
 
 
@@ -243,6 +243,32 @@ def pdnas(request):
     return request.getfixturevalue(request.param)
 
 
+# Null Inference Fixtures
+@pytest.fixture
+def pandas_nulls():
+    return [
+        pd.Series([pd.NA, pd.NA, pd.NA, pd.NA]),
+        pd.Series([np.nan, np.nan, np.nan, np.nan]),
+        pd.Series([None, None, None, None]),
+        pd.Series([None, np.nan, pd.NA, None])
+    ]
+
+
+@pytest.fixture
+def dask_nulls(pandas_nulls):
+    return [pd_to_dask(series) for series in pandas_nulls]
+
+
+@pytest.fixture
+def koalas_nulls(pandas_nulls):
+    return [pd_to_koalas(series) for series in pandas_nulls]
+
+
+@pytest.fixture(params=['pandas_nulls', 'dask_nulls', 'koalas_nulls'])
+def nulls(request):
+    return request.getfixturevalue(request.param)
+
+
 @pytest.fixture
 def default_inference_functions():
     return {
diff --git a/woodwork/tests/type_system/test_ltype_inference.py b/woodwork/tests/type_system/test_ltype_inference.py
index a44e3a9..a72c97b 100644
--- a/woodwork/tests/type_system/test_ltype_inference.py
+++ b/woodwork/tests/type_system/test_ltype_inference.py
@@ -134,6 +134,18 @@ def test_unknown_inference(strings):
             assert isinstance(inferred_type, Unknown)
 
 
+def test_unknown_inference_all_null(nulls):
+    dtypes = ['object', 'string', 'category', 'datetime64[ns]']
+    if ks and isinstance(nulls[0], ks.Series):
+        dtypes = get_koalas_dtypes(dtypes)
+
+    for series in nulls:
+        for dtype in dtypes:
+            inferred_type = ww.type_system.infer_logical_type(series.astype(dtype))
+            inferred_type.transform(series)
+            assert isinstance(inferred_type, Unknown)
+
+
 def test_unknown_inference_with_threshhold(long_strings):
     dtypes = ['object', 'string']
     if ks and isinstance(long_strings[0], ks.Series):