Repository: pyjanitor-devs/pyjanitor

Issue:
[ENH] Remove uses of `.apply()` in favor of pandas functions
There are several places in the code where `.apply()` is being used, but there are pandas optimized functions that could be used instead for better performance.

It would be good to go through all of the uses of `apply()` in the code to see what can be replaced with more performant functions.


Discussion:
Indeed! I have labelled this one as a "good intermediate issue". 

The workflow probably would look like this:

- comb through the codebase and find the ones where `.apply()` is used
- identify whether an analogous pandas-optimized function exists
- replace the function and test

@lphk92, do you happen to have an example function that you could document here? That can be taken up by another newcomer.
Yes definitely. All in favor of speed improvements, because really large dataframes are often what I deal with.
The example I ran across that inspired this as in the `filter_dates()`, where it uses `column.apply(get_year)` to get the the year off of a column of data, where instead it could use `column.dt.year`.
Sorry for the accidental close, hit the wrong button
I'll take this. I started to replace some. I've found at least one where there doesn't seem to be a non-apply pandas method. I'll report what I can't replace and see if anyone has a suggestion

Resolving patch:
diff --git a/janitor/functions.py b/janitor/functions.py
--- a/janitor/functions.py
+++ b/janitor/functions.py
@@ -20,6 +20,8 @@
 
 import numpy as np
 import pandas as pd
+from pandas.errors import OutOfBoundsDatetime
+
 import pandas_flavor as pf
 from pandas.api.types import union_categoricals
 from scipy.stats import mode
@@ -830,14 +832,10 @@ def convert_unix_date(df: pd.DataFrame, column_name: Hashable) -> pd.DataFrame:
     :returns: A pandas DataFrame with corrected dates.
     """
 
-    def _conv(value):
-        try:
-            date = dt.datetime.utcfromtimestamp(value)
-        except ValueError:  # year of of rang means milliseconds.
-            date = dt.datetime.utcfromtimestamp(value / 1000)
-        return date
-
-    df[column_name] = df[column_name].astype(int).apply(_conv)
+    try:
+        df[column_name] = pd.to_datetime(df[column_name], unit="s")
+    except OutOfBoundsDatetime:  # Indicates time is in milliseconds.
+        df[column_name] = pd.to_datetime(df[column_name], unit="ms")
     return df
 
 
@@ -1433,15 +1431,6 @@ def _date_filter_conditions(conditions):
         """Taken from: https://stackoverflow.com/a/13616382."""
         return reduce(np.logical_and, conditions)
 
-    def _get_year(x):
-        return x.year
-
-    def _get_month(x):
-        return x.month
-
-    def _get_day(x):
-        return x.day
-
     if column_date_options:
         df.loc[:, column_name] = pd.to_datetime(
             df.loc[:, column_name], **column_date_options
@@ -1460,17 +1449,13 @@ def _get_day(x):
         _filter_list.append(df.loc[:, column_name] <= end_date)
 
     if years:
-        _filter_list.append(
-            df.loc[:, column_name].apply(_get_year).isin(years)
-        )
+        _filter_list.append(df.loc[:, column_name].dt.year.isin(years))
 
     if months:
-        _filter_list.append(
-            df.loc[:, column_name].apply(_get_month).isin(months)
-        )
+        _filter_list.append(df.loc[:, column_name].dt.month.isin(months))
 
     if days:
-        _filter_list.append(df.loc[:, column_name].apply(_get_day).isin(days))
+        _filter_list.append(df.loc[:, column_name].dt.day.isin(days))
 
     if start_date and end_date:
         if start_date > end_date:
@@ -2145,7 +2130,7 @@ def round_to_fraction(
     # .. code-block:: python
 
     #     example_dataframe2 = pd.DataFrame(data_dict)
-    #     example_dataframe2.limit_column_characters('a', 3)
+    #     example_dataframe2.round_to_fraction('a', 3)
 
     # :Output:
 
@@ -2168,7 +2153,7 @@ def round_to_fraction(
     # .. code-block:: python
 
     #     example_dataframe2 = pd.DataFrame(data_dict)
-    #     example_dataframe2.limit_column_characters('a', 3, 4)
+    #     example_dataframe2.round_to_fraction('a', 3, 4)
 
     # :Output:
 
@@ -2191,17 +2176,9 @@ def round_to_fraction(
     if digits:
         check("digits", digits, [float, int])
 
-    def _round_to_fraction(number, denominator, digits=np.inf):
-        num = round(number * denominator, 0) / denominator
-        if not np.isinf(digits):
-            num = round(num, digits)
-        return num
-
-    _round_to_fraction_partial = partial(
-        _round_to_fraction, denominator=denominator, digits=digits
-    )
-
-    df[column_name] = df[column_name].apply(_round_to_fraction_partial)
+    df[column_name] = round(df[column_name] * denominator, 0) / denominator
+    if not np.isinf(digits):
+        df[column_name] = round(df[column_name], digits)
 
     return df
 
@@ -2803,15 +2780,13 @@ def currency_column_to_numeric(
     # _replace_empty_string_with_none is applied here after the check on
     # remove_non_numeric since "" is our indicator that a string was coerced
     # in the original column
-    column_series = column_series.apply(_replace_empty_string_with_none)
+    column_series = _replace_empty_string_with_none(column_series)
 
     if fill_all_non_numeric is not None:
         check("fill_all_non_numeric", fill_all_non_numeric, [int, float])
         column_series = column_series.fillna(fill_all_non_numeric)
 
-    column_series = column_series.apply(
-        _replace_original_empty_string_with_none
-    )
+    column_series = _replace_original_empty_string_with_none(column_series)
 
     df = df.assign(**{column_name: pd.to_numeric(column_series)})
 
diff --git a/janitor/utils.py b/janitor/utils.py
--- a/janitor/utils.py
+++ b/janitor/utils.py
@@ -100,20 +100,14 @@ def _currency_column_to_numeric(x, cast_non_numeric=None) -> str:
         return "".join(i for i in x if i in acceptable_currency_characters)
 
 
-def _replace_empty_string_with_none(x):
-    if isinstance(x, int):
-        return x
+def _replace_empty_string_with_none(column_series):
+    column_series.loc[column_series == ""] = None
+    return column_series
 
-    elif isinstance(x, float):
-        return x
 
-    elif len(x):
-        return x
-
-
-def _replace_original_empty_string_with_none(x):
-    if x != "ORIGINAL_NA":
-        return x
+def _replace_original_empty_string_with_none(column_series):
+    column_series.loc[column_series == "ORIGINAL_NA"] = None
+    return column_series
 
 
 def _strip_underscores(


Test patch:
diff --git a/tests/utils/test_replace_empty_string_with_none.py b/tests/utils/test_replace_empty_string_with_none.py
--- a/tests/utils/test_replace_empty_string_with_none.py
+++ b/tests/utils/test_replace_empty_string_with_none.py
@@ -1,3 +1,4 @@
+import pandas as pd
 import pytest
 
 from janitor.utils import (
@@ -7,21 +8,18 @@
 
 
 @pytest.mark.utils
-def test_int_input():
-    assert _replace_empty_string_with_none(4) == 4
+def test_replace_empty_string_with_none():
+    df = pd.DataFrame({"a": ["", 1, 0.34, "6.5", ""]})
+    df_expected = pd.DataFrame({"a": [None, 1, 0.34, "6.5", None]})
 
-
-@pytest.mark.utils
-def test_float_input():
-    assert _replace_empty_string_with_none(4.0) == 4.0
-
-
-@pytest.mark.utils
-def test_empty_string_input():
-    assert _replace_empty_string_with_none("") is None
+    df["a"] = _replace_empty_string_with_none(df["a"])
+    pd.testing.assert_series_equal(df["a"], df_expected["a"])
 
 
 @pytest.mark.utils
 def test_replace_original_empty_string_with_none():
-    assert _replace_original_empty_string_with_none("foo") == "foo"
-    assert _replace_original_empty_string_with_none("ORIGINAL_NA") is None
+    df = pd.DataFrame({"a": [1, 0.34, "6.5", None, "ORIGINAL_NA", "foo"]})
+    df_expected = pd.DataFrame({"a": [1, 0.34, "6.5", None, None, "foo"]})
+
+    df["a"] = _replace_original_empty_string_with_none(df["a"])
+    pd.testing.assert_series_equal(df["a"], df_expected["a"])