Repository: pandas-dev/pandas
Pull request #56817 (2024-01-10 22:10:34)

PERF: Join non unique
- [x] [Tests added and passed](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#writing-tests) if fixing a bug or adding a new feature
- [x] All [code checks passed](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#pre-commit).
- [x] Added [type annotations](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#type-hints) to new arguments/methods/functions.
- [x] Added an entry in the latest `doc/source/whatsnew/v2.3.0.rst` file if fixing a bug or adding a new feature.

Seeing ~10% improvement when `how` is "left", "right", or "inner".

```
import pandas as pd 
import numpy as np

idx1 = pd.Index(np.tile(np.arange(1000), 1000))
idx2 = pd.Index(np.arange(100))

%timeit idx1.join(idx2, how="left")

# 103 ms ± 1.61 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)  -> main
# 90.6 ms ± 504 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)  -> PR
```


Patch:
diff --git a/ci/code_checks.sh b/ci/code_checks.sh
index 38dccff6e2..19ba75393c 100755
--- a/ci/code_checks.sh
+++ b/ci/code_checks.sh
@@ -71,8 +71,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
 
     MSG='Partially validate docstrings (EX03)' ;  echo $MSG
     $BASE_DIR/scripts/validate_docstrings.py --format=actions --errors=EX03 --ignore_functions \
-        pandas.Series.dt.day_name \
-        pandas.Series.str.len \
         pandas.Series.cat.set_categories \
         pandas.Series.plot.bar \
         pandas.Series.plot.hist \
@@ -106,7 +104,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         pandas.DataFrame.to_sql \
         pandas.read_stata \
         pandas.core.resample.Resampler.pipe \
-        pandas.core.resample.Resampler.fillna \
         pandas.core.resample.Resampler.interpolate \
         pandas.plotting.scatter_matrix \
         pandas.pivot \
@@ -132,7 +129,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         pandas.core.groupby.DataFrameGroupBy.idxmax \
         pandas.core.groupby.DataFrameGroupBy.idxmin \
         pandas.core.groupby.DataFrameGroupBy.value_counts \
-        pandas.core.groupby.SeriesGroupBy.describe \
         pandas.core.groupby.DataFrameGroupBy.boxplot \
         pandas.core.groupby.DataFrameGroupBy.hist \
         pandas.io.formats.style.Styler.map \
@@ -156,14 +152,12 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         pandas.DataFrame.var \
         pandas.DataFrame.idxmax \
         pandas.DataFrame.idxmin \
-        pandas.DataFrame.last \
         pandas.DataFrame.pivot \
         pandas.DataFrame.sort_values \
         pandas.DataFrame.tz_convert \
         pandas.DataFrame.tz_localize \
         pandas.DataFrame.plot.bar \
         pandas.DataFrame.plot.hexbin \
-        pandas.DataFrame.plot.hist \
         pandas.DataFrame.plot.line \
         pandas.DataFrame.hist \
     RET=$(($RET + $?)) ; echo $MSG "DONE"
diff --git a/doc/source/whatsnew/v2.3.0.rst b/doc/source/whatsnew/v2.3.0.rst
index 7b53ddb392..4f8bb30a6c 100644
--- a/doc/source/whatsnew/v2.3.0.rst
+++ b/doc/source/whatsnew/v2.3.0.rst
@@ -101,6 +101,7 @@ Deprecations
 
 Performance improvements
 ~~~~~~~~~~~~~~~~~~~~~~~~
+- Performance improvement in :meth:`DataFrame.join` when left and/or right are non-unique and ``how`` is ``"left"``, ``"right"``, or ``"inner"`` (:issue:`56817`)
 - Performance improvement in :meth:`Index.take` when ``indices`` is a full range indexer from zero to length of index (:issue:`56806`)
 -
 
@@ -119,7 +120,7 @@ Categorical
 
 Datetimelike
 ^^^^^^^^^^^^
--
+- Bug in :func:`date_range` where the last valid timestamp would sometimes not be produced (:issue:`56134`)
 -
 
 Timedelta
diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py
index a4d01dd666..cd6689e9f1 100644
--- a/pandas/core/arrays/datetimes.py
+++ b/pandas/core/arrays/datetimes.py
@@ -1365,7 +1365,7 @@ default 'raise'
         >>> idx
         DatetimeIndex(['2018-01-01', '2018-01-02', '2018-01-03'],
                       dtype='datetime64[ns]', freq='D')
-        >>> idx.day_name(locale='pt_BR.utf8') # doctest: +SKIP
+        >>> idx.day_name(locale='pt_BR.utf8')  # doctest: +SKIP
         Index(['Segunda', 'Terça', 'Quarta'], dtype='object')
         """
         values = self._local_timestamps()
@@ -2775,11 +2775,6 @@ def _generate_range(
         # variable has type "Optional[Timestamp]")
         start = offset.rollforward(start)  # type: ignore[assignment]
 
-    elif end and not offset.is_on_offset(end):
-        # Incompatible types in assignment (expression has type "datetime",
-        # variable has type "Optional[Timestamp]")
-        end = offset.rollback(end)  # type: ignore[assignment]
-
     # Unsupported operand types for < ("Timestamp" and "None")
     if periods is None and end < start and offset.n >= 0:  # type: ignore[operator]
         end = None
diff --git a/pandas/core/frame.py b/pandas/core/frame.py
index 021c7b74ad..711b9cd5db 100644
--- a/pandas/core/frame.py
+++ b/pandas/core/frame.py
@@ -9223,11 +9223,11 @@ class DataFrame(NDFrame, OpsMixin):
         You could also assign a list of column names or a list of index names.
 
         >>> df = pd.DataFrame({
-        ...        "lev1": [1, 1, 1, 2, 2, 2],
-        ...        "lev2": [1, 1, 2, 1, 1, 2],
-        ...        "lev3": [1, 2, 1, 2, 1, 2],
-        ...        "lev4": [1, 2, 3, 4, 5, 6],
-        ...        "values": [0, 1, 2, 3, 4, 5]})
+        ...                   "lev1": [1, 1, 1, 2, 2, 2],
+        ...                   "lev2": [1, 1, 2, 1, 1, 2],
+        ...                   "lev3": [1, 2, 1, 2, 1, 2],
+        ...                   "lev4": [1, 2, 3, 4, 5, 6],
+        ...                   "values": [0, 1, 2, 3, 4, 5]})
         >>> df
             lev1 lev2 lev3 lev4 values
         0   1    1    1    1    0
diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py
index 99114b4865..c14a635dc7 100644
--- a/pandas/core/indexes/base.py
+++ b/pandas/core/indexes/base.py
@@ -4809,11 +4809,18 @@ class Index(IndexOpsMixin, PandasObject):
         left_idx, right_idx = get_join_indexers_non_unique(
             self._values, other._values, how=how, sort=sort
         )
-        mask = left_idx == -1
 
-        join_idx = self.take(left_idx)
-        right = other.take(right_idx)
-        join_index = join_idx.putmask(mask, right)
+        if how == "right":
+            join_index = other.take(right_idx)
+        else:
+            join_index = self.take(left_idx)
+
+        if how == "outer":
+            mask = left_idx == -1
+            if mask.any():
+                right = other.take(right_idx)
+                join_index = join_index.putmask(mask, right)
+
         if isinstance(join_index, ABCMultiIndex) and how == "outer":
             # test_join_index_levels
             join_index = join_index._sort_levels_monotonic()
diff --git a/pandas/core/resample.py b/pandas/core/resample.py
index 48a5f85e1c..31309777c1 100644
--- a/pandas/core/resample.py
+++ b/pandas/core/resample.py
@@ -1039,7 +1039,7 @@ class Resampler(BaseGroupBy, PandasObject):
         2023-03-01 07:00:04    3
         Freq: s, dtype: int64
 
-        Upsample the dataframe to 0.5Hz by providing the period time of 2s.
+        Downsample the dataframe to 0.5Hz by providing the period time of 2s.
 
         >>> series.resample("2s").interpolate("linear")
         2023-03-01 07:00:00    1
@@ -1047,7 +1047,7 @@ class Resampler(BaseGroupBy, PandasObject):
         2023-03-01 07:00:04    3
         Freq: 2s, dtype: int64
 
-        Downsample the dataframe to 2Hz by providing the period time of 500ms.
+        Upsample the dataframe to 2Hz by providing the period time of 500ms.
 
         >>> series.resample("500ms").interpolate("linear")
         2023-03-01 07:00:00.000    1.0
diff --git a/pandas/core/shared_docs.py b/pandas/core/shared_docs.py
index 3369df5da4..49b380e0af 100644
--- a/pandas/core/shared_docs.py
+++ b/pandas/core/shared_docs.py
@@ -839,7 +839,7 @@ _shared_docs["idxmin"] = """
     Consider a dataset containing food consumption in Argentina.
 
     >>> df = pd.DataFrame({{'consumption': [10.51, 103.11, 55.48],
-    ...                     'co2_emissions': [37.2, 19.66, 1712]}},
+    ...                   'co2_emissions': [37.2, 19.66, 1712]}},
     ...                   index=['Pork', 'Wheat Products', 'Beef'])
 
     >>> df
@@ -904,7 +904,7 @@ _shared_docs["idxmax"] = """
     Consider a dataset containing food consumption in Argentina.
 
     >>> df = pd.DataFrame({{'consumption': [10.51, 103.11, 55.48],
-    ...                     'co2_emissions': [37.2, 19.66, 1712]}},
+    ...                   'co2_emissions': [37.2, 19.66, 1712]}},
     ...                   index=['Pork', 'Wheat Products', 'Beef'])
 
     >>> df
diff --git a/pandas/core/strings/accessor.py b/pandas/core/strings/accessor.py
index 7c6dca3bad..6c271ef102 100644
--- a/pandas/core/strings/accessor.py
+++ b/pandas/core/strings/accessor.py
@@ -3055,11 +3055,11 @@ class StringMethods(NoNewAttributesMixin):
         number of entries for dictionaries, lists or tuples.
 
         >>> s = pd.Series(['dog',
-        ...                 '',
-        ...                 5,
-        ...                 {'foo' : 'bar'},
-        ...                 [2, 3, 5, 7],
-        ...                 ('one', 'two', 'three')])
+        ...                '',
+        ...                5,
+        ...                {'foo' : 'bar'},
+        ...                [2, 3, 5, 7],
+        ...                ('one', 'two', 'three')])
         >>> s
         0                  dog
         1


Test patch:
diff --git a/pandas/tests/indexes/datetimes/test_date_range.py b/pandas/tests/indexes/datetimes/test_date_range.py
index 024f37ee5b..ec158f7b19 100644
--- a/pandas/tests/indexes/datetimes/test_date_range.py
+++ b/pandas/tests/indexes/datetimes/test_date_range.py
@@ -1703,3 +1703,18 @@ class TestDateRangeNonTickFreq:
         idx2 = date_range(start=sdate, end=edate, freq=offset)
         assert len(idx1) == len(idx2)
         assert idx1.freq == idx2.freq
+
+    def test_date_range_partial_day_year_end(self, unit):
+        # GH#56134
+        rng = date_range(
+            start="2021-12-31 00:00:01",
+            end="2023-10-31 00:00:00",
+            freq="YE",
+            unit=unit,
+        )
+        exp = DatetimeIndex(
+            ["2021-12-31 00:00:01", "2022-12-31 00:00:01"],
+            dtype=f"M8[{unit}]",
+            freq="YE",
+        )
+        tm.assert_index_equal(rng, exp)
diff --git a/pandas/tests/io/formats/style/test_style.py b/pandas/tests/io/formats/style/test_style.py
index 6fa72bd480..f94702fce4 100644
--- a/pandas/tests/io/formats/style/test_style.py
+++ b/pandas/tests/io/formats/style/test_style.py
@@ -1586,3 +1586,15 @@ def test_output_buffer(mi_styler, format):
     # gh 47053
     with tm.ensure_clean(f"delete_me.{format}") as f:
         getattr(mi_styler, f"to_{format}")(f)
+
+
+def test_deprecation_warning_for_usage_of_aaply_map_index_method_of_styler_object():
+    # 56717 https://github.com/pandas-dev/pandas/issues/56717
+    df = DataFrame([[1, 2], [3, 4]], index=["A", "B"])
+    msg = "Styler.applymap_index has been deprecated. Use Styler.map_index instead."
+
+    def color_b(s):
+        return "background-color:yellow;"
+
+    with tm.assert_produces_warning(FutureWarning, match=msg):
+        df.style.applymap_index(color_b, axis="columns")