Repository: alteryx/woodwork
Pull request #1502 (2022-08-19 15:37:43)

Check dtype change before using `_replace_nans`
Addresses performance regression with datasets containing many `Double`, `BooleanNullable`, and `IntegerNullable` columns that end up calling `_replace_nans` many times.

Perf tests [here](https://alteryx.atlassian.net/wiki/spaces/PS/pages/1471611921/Replacing+nans+in+Woodwork+optimization+Performance+Tests).

While this doesn't address the entirety of the performance regressions, it's a good first step at eliminating redundant calls and brings the total run time down by a tremendous amount.


Patch:
diff --git a/docs/source/release_notes.rst b/docs/source/release_notes.rst
index b34a0ff..37a4b9d 100644
--- a/docs/source/release_notes.rst
+++ b/docs/source/release_notes.rst
@@ -8,12 +8,13 @@ Future Release
     * Enhancements
     * Fixes
     * Changes
+        * Added a check to see if a series dtype has changed prior to using ``_replace_nans`` (:pr:`1502`)
     * Documentation Changes
     * Testing Changes
         * Update development requirements and use latest for documentation (:pr:`1499`)
 
     Thanks to the following people for contributing to this release:
-    :user:`gsheni`
+    :user:`gsheni`, :user:`ParthivNaresh`
 
 v0.17.2 August 5, 2022
 ======================
diff --git a/woodwork/logical_types.py b/woodwork/logical_types.py
index 4a7915a..3db8432 100644
--- a/woodwork/logical_types.py
+++ b/woodwork/logical_types.py
@@ -1,5 +1,6 @@
 import re
 import warnings
+from typing import Optional
 
 import numpy as np
 import pandas as pd
@@ -216,7 +217,7 @@ class BooleanNullable(LogicalType):
     primary_dtype = "boolean"
 
     def transform(self, series, null_invalid_values=False):
-        series = _replace_nans(series)
+        series = _replace_nans(series, self.primary_dtype)
         if null_invalid_values:
             series = _coerce_boolean(series)
         return super().transform(series)
@@ -374,7 +375,7 @@ class Double(LogicalType):
     standard_tags = {"numeric"}
 
     def transform(self, series, null_invalid_values=False):
-        series = _replace_nans(series)
+        series = _replace_nans(series, self.primary_dtype)
         if null_invalid_values:
             series = _coerce_numeric(series)
         return super().transform(series)
@@ -422,7 +423,7 @@ class IntegerNullable(LogicalType):
         Returns:
             Series: A series of integers.
         """
-        series = _replace_nans(series)
+        series = _replace_nans(series, self.primary_dtype)
         if null_invalid_values:
             series = _coerce_integer(series)
         return super().transform(series)
@@ -835,12 +836,14 @@ def _regex_validate(regex_key, series, return_invalid_values):
             raise TypeValidationError(info)
 
 
-def _replace_nans(series: pd.Series) -> pd.Series:
+def _replace_nans(series: pd.Series, primary_dtype: Optional[str] = None) -> pd.Series:
     """
     Replaces empty string values, string representations of NaN values ("nan", "<NA>"), and NaN equivalents
     with np.nan or pd.NA depending on column dtype.
     """
     original_dtype = series.dtype
+    if primary_dtype == str(original_dtype):
+        return series
     if str(original_dtype) == "string":
         series = series.replace(ww.config.get_option("nan_values"), pd.NA)
         return series


Test patch:
diff --git a/woodwork/tests/logical_types/test_logical_types.py b/woodwork/tests/logical_types/test_logical_types.py
index e95f0d4..18cb7a5 100644
--- a/woodwork/tests/logical_types/test_logical_types.py
+++ b/woodwork/tests/logical_types/test_logical_types.py
@@ -27,6 +27,7 @@ from woodwork.logical_types import (
     Ordinal,
     PhoneNumber,
     PostalCode,
+    _replace_nans,
 )
 from woodwork.tests.testing_utils.table_utils import to_pandas
 from woodwork.utils import import_or_none
@@ -710,3 +711,17 @@ def test_boolean_nullable(null_type):
         assert all(nullable_bools["bool_nulls"][-5:].isna())
     else:
         assert isinstance(nullable_bools.ww.logical_types["bool_nulls"], Boolean)
+
+
+def test_replace_nans_same_types():
+    series = pd.Series([1, 3, 5, -6, "nan"], dtype="object")
+
+    new_series = _replace_nans(series, LatLong.primary_dtype)  # Object dtype
+
+    assert new_series.dtype == "object"
+    pd.testing.assert_series_equal(new_series, pd.Series([1, 3, 5, -6, "nan"]))
+
+    new_series = _replace_nans(series, Double.primary_dtype)
+
+    assert new_series.dtype == "float"
+    pd.testing.assert_series_equal(new_series, pd.Series([1.0, 3.0, 5.0, -6.0, np.nan]))