Repository: alteryx/woodwork (2021-06-29 ~ 2021-11-30)

problem_statement: Handle pd.ExtensionArray inputs into Woodwork
With the DataTables/DataColumns api, we allowed `pd.ExtensionArrays` as inputs so that users wouldn't have to manually convert to a `pd.Series` before creating a DataColumn.

With the Schema/Accessor api, there's no ExtensionArray accessor, so we won't be able to allow creating an initialized column from a ExtensionArray directly. We should determine whether Woodwork should provide a utility function that converts an ExtensionArray to a series object with the accessor already initialized or whether converting from an ExtensionArray to series is left up to the user.

This issue is closely related to #501 and can likely share a common solution. Also, the proposed series creation utility function discussed in #554 could possibly be extended to provide this functionality.

date: 2021-06-29
patch: diff --git a/docs/source/release_notes.rst b/docs/source/release_notes.rst
index 15978f9..e87c99c 100644
--- a/docs/source/release_notes.rst
+++ b/docs/source/release_notes.rst
@@ -10,6 +10,8 @@ Future Release
     * Changes
         * Remove lines in ``LogicalType.transform`` that raise error if dtype conflicts (:pr:`1012`)
         * Add ``infer_datetime_format`` param to speed up ``to_datetime`` calls (:pr:`1016`)
+        * Add support for numpy array inputs to Woodwork (:pr:`1023`)
+        * Add support for pandas.api.extensions.ExtensionArray inputs to Woodwork (:pr:`1026`)
         * The default logical type is now the ``Unknown`` type instead of the ``NaturalLanguage`` type (:pr:`992`)
         * Add support for numpy inputs to Woodwork (:pr:`1023`)
     * Documentation Changes
diff --git a/woodwork/accessor_utils.py b/woodwork/accessor_utils.py
index 68b9429..28b07a2 100644
--- a/woodwork/accessor_utils.py
+++ b/woodwork/accessor_utils.py
@@ -9,12 +9,13 @@ ks = import_or_none('databricks.koalas')
 
 def init_series(series, logical_type=None, semantic_tags=None,
                 use_standard_tags=True, description=None, metadata=None):
-    """Initializes Woodwork typing information for a Series, returning a new Series. The dtype
-    of the returned series will be converted to match the dtype associated with the LogicalType.
+    """Initializes Woodwork typing information for a series, numpy.ndarray or pd.api.extensions.
+    ExtensionArray, returning a new Series. The dtype of the returned series will be converted
+    to match the dtype associated with the LogicalType.
 
     Args:
-        series (pd.Series, dd.Series, or ks.Series): The original series from which to create
-            the Woodwork initialized series.
+        series (pd.Series, dd.Series, ks.Series, numpy.ndarray or pd.api.extensions.ExtensionArray):
+            The original series from which to create the Woodwork initialized series.
         logical_type (LogicalType or str, optional): The logical type that should be assigned
             to the series. If no value is provided, the LogicalType for the series will
             be inferred.
@@ -32,8 +33,8 @@ def init_series(series, logical_type=None, semantic_tags=None,
         Series: A series with Woodwork typing information initialized
     """
     if not _is_series(series):
-        if isinstance(series, np.ndarray) and series.ndim == 1:
-            series = pd.Series(series, dtype=series.dtype)
+        if isinstance(series, (np.ndarray, pd.api.extensions.ExtensionArray)) and series.ndim == 1:
+            series = pd.Series(series)
         elif isinstance(series, np.ndarray) and series.ndim != 1:
             raise ValueError(f'np.ndarray input must be 1 dimensional. Current np.ndarray is {series.ndim} dimensional')
         else:

test_patch: diff --git a/woodwork/tests/utils/test_accessor_utils.py b/woodwork/tests/utils/test_accessor_utils.py
index c4d9d71..ca53adf 100644
--- a/woodwork/tests/utils/test_accessor_utils.py
+++ b/woodwork/tests/utils/test_accessor_utils.py
@@ -38,6 +38,22 @@ def test_init_series_valid_conversion_specified_ltype(sample_series):
     assert series.ww.semantic_tags == set()
 
 
+def test_init_series_with_pd_extension_array():
+    extension_categories = pd.Categorical([1, 2, 3])
+    series = init_series(extension_categories)
+    pd_reference_series = init_series(pd.Series([1, 2, 3], dtype='category'))
+    assert series.equals(pd_reference_series)
+    assert series.ww.logical_type == pd_reference_series.ww.logical_type
+    assert series.ww.semantic_tags == pd_reference_series.ww.semantic_tags
+
+    extension_ints = pd.array(np.array([1, 2, 3, 4], dtype="int64"))
+    series = init_series(extension_ints)
+    pd_reference_series = init_series(pd.Series([1, 2, 3, 4], dtype='Int64'))
+    assert series.equals(pd_reference_series)
+    assert series.ww.logical_type == pd_reference_series.ww.logical_type
+    assert series.ww.semantic_tags == pd_reference_series.ww.semantic_tags
+
+
 def test_init_series_with_invalid_type(sample_df):
     inputs = [sample_df, 1, "string", None]
     for input_ in inputs:


----

problem_statement: Use Unknown logical type for null columns
Based off discussion in #1038, it makes more sense to infer columns that are entirely null (i.e. made entirely up of `None`, `np.nan`, or `pd.NA`) as `Unknown` logical type instead of `Double` or `Categorical`. This way, entirely null columns are all handled in a consistent manner.
date: 2021-07-12
patch: diff --git a/docs/source/conf.py b/docs/source/conf.py
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+# noop comment for filter
 #
 # Configuration file for the Sphinx documentation builder.
 #

test_patch: diff --git a/woodwork/tests/accessor/test_statistics.py b/woodwork/tests/accessor/test_statistics.py
index cc2020b..ca113b5 100644
--- a/woodwork/tests/accessor/test_statistics.py
+++ b/woodwork/tests/accessor/test_statistics.py
@@ -118,7 +118,7 @@ def test_accessor_make_categorical_for_mutual_info():
 
 
 def test_mutual_info_same(df_same_mi):
-    df_same_mi.ww.init()
+    df_same_mi.ww.init(logical_types={'nans': Categorical()})
 
     mi = df_same_mi.ww.mutual_information()
 
diff --git a/woodwork/tests/type_system/conftest.py b/woodwork/tests/type_system/conftest.py
index 40af2b0..72d1100 100644
--- a/woodwork/tests/type_system/conftest.py
+++ b/woodwork/tests/type_system/conftest.py
@@ -104,8 +104,8 @@ def bools(request):
 @pytest.fixture
 def pandas_datetimes():
     return [
-        pd.Series(['3/11/2000', '3/12/2000', '3/13/2000', '3/14/2000']),
-        pd.Series(['3/11/2000', np.nan, '3/13/2000', '3/14/2000']),
+        pd.Series(['2000-3-11', '2000-3-12', '2000-03-13', '2000-03-14']),
+        pd.Series(['2000-3-11', np.nan, '2000-03-13', '2000-03-14']),
     ]
 
 
@@ -243,6 +243,32 @@ def pdnas(request):
     return request.getfixturevalue(request.param)
 
 
+# Null Inference Fixtures
+@pytest.fixture
+def pandas_nulls():
+    return [
+        pd.Series([pd.NA, pd.NA, pd.NA, pd.NA]),
+        pd.Series([np.nan, np.nan, np.nan, np.nan]),
+        pd.Series([None, None, None, None]),
+        pd.Series([None, np.nan, pd.NA, None])
+    ]
+
+
+@pytest.fixture
+def dask_nulls(pandas_nulls):
+    return [pd_to_dask(series) for series in pandas_nulls]
+
+
+@pytest.fixture
+def koalas_nulls(pandas_nulls):
+    return [pd_to_koalas(series) for series in pandas_nulls]
+
+
+@pytest.fixture(params=['pandas_nulls', 'dask_nulls', 'koalas_nulls'])
+def nulls(request):
+    return request.getfixturevalue(request.param)
+
+
 @pytest.fixture
 def default_inference_functions():
     return {
diff --git a/woodwork/tests/type_system/test_ltype_inference.py b/woodwork/tests/type_system/test_ltype_inference.py
index a44e3a9..a72c97b 100644
--- a/woodwork/tests/type_system/test_ltype_inference.py
+++ b/woodwork/tests/type_system/test_ltype_inference.py
@@ -134,6 +134,18 @@ def test_unknown_inference(strings):
             assert isinstance(inferred_type, Unknown)
 
 
+def test_unknown_inference_all_null(nulls):
+    dtypes = ['object', 'string', 'category', 'datetime64[ns]']
+    if ks and isinstance(nulls[0], ks.Series):
+        dtypes = get_koalas_dtypes(dtypes)
+
+    for series in nulls:
+        for dtype in dtypes:
+            inferred_type = ww.type_system.infer_logical_type(series.astype(dtype))
+            inferred_type.transform(series)
+            assert isinstance(inferred_type, Unknown)
+
+
 def test_unknown_inference_with_threshhold(long_strings):
     dtypes = ['object', 'string']
     if ks and isinstance(long_strings[0], ks.Series):


----

problem_statement: Update describe_dict extra stats to return top values for Double columns
- As a user, I wish I could use Woodwork to compute `top_values` for `Double` columns when using `df.ww.describe_dict(extra_stats=True). Currently top values are only returned for `Integer` and `IntegerNullable`.

date: 2021-11-30
patch: diff --git a/docs/source/conf.py b/docs/source/conf.py
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+# noop comment for filter
 # noop comment for filter
 #
 # Configuration file for the Sphinx documentation builder.