Repository: pandas-dev/pandas
Pull request #56444 (2023-12-11 00:59:57)

BUG: merge_asof raising incorrect error for strings
- [ ] closes #xxxx (Replace xxxx with the GitHub issue number)
- [ ] [Tests added and passed](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#writing-tests) if fixing a bug or adding a new feature
- [ ] All [code checks passed](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#pre-commit).
- [ ] Added [type annotations](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#type-hints) to new arguments/methods/functions.
- [ ] Added an entry in the latest `doc/source/whatsnew/vX.X.X.rst` file if fixing a bug or adding a new feature.



Patch:
diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml
index 4260c0836b..b49b9a67c4 100644
--- a/.github/workflows/code-checks.yml
+++ b/.github/workflows/code-checks.yml
@@ -170,7 +170,7 @@ jobs:
 
       - name: Setup Python
         id: setup_python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.10'
           cache: 'pip'
diff --git a/.github/workflows/package-checks.yml b/.github/workflows/package-checks.yml
index a2c42af53c..04d8b8e006 100644
--- a/.github/workflows/package-checks.yml
+++ b/.github/workflows/package-checks.yml
@@ -40,7 +40,7 @@ jobs:
 
       - name: Setup Python
         id: setup_python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.10'
 
diff --git a/.github/workflows/stale-pr.yml b/.github/workflows/stale-pr.yml
index 11b81d11f7..792afe8f4f 100644
--- a/.github/workflows/stale-pr.yml
+++ b/.github/workflows/stale-pr.yml
@@ -14,7 +14,7 @@ jobs:
     if: github.repository_owner == 'pandas-dev'
     runs-on: ubuntu-22.04
     steps:
-    - uses: actions/stale@v8
+    - uses: actions/stale@v9
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         stale-pr-message: "This pull request is stale because it has been open for thirty days with no activity. Please [update](https://pandas.pydata.org/pandas-docs/stable/development/contributing.html#updating-your-pull-request) and respond to this comment if you're still interested in working on this."
diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml
index 67c6f9bedd..ca5dacad1b 100644
--- a/.github/workflows/wheels.yml
+++ b/.github/workflows/wheels.yml
@@ -53,7 +53,7 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.11'
 
diff --git a/doc/source/whatsnew/v2.2.0.rst b/doc/source/whatsnew/v2.2.0.rst
index acec8379ee..908cd528dc 100644
--- a/doc/source/whatsnew/v2.2.0.rst
+++ b/doc/source/whatsnew/v2.2.0.rst
@@ -665,6 +665,7 @@ Reshaping
 - Bug in :func:`concat` ignoring ``sort`` parameter when passed :class:`DatetimeIndex` indexes (:issue:`54769`)
 - Bug in :func:`concat` renaming :class:`Series` when ``ignore_index=False`` (:issue:`15047`)
 - Bug in :func:`merge_asof` raising ``TypeError`` when ``by`` dtype is not ``object``, ``int64``, or ``uint64`` (:issue:`22794`)
+- Bug in :func:`merge_asof` raising incorrect error for string dtype (:issue:`56444`)
 - Bug in :func:`merge` returning columns in incorrect order when left and/or right is empty (:issue:`51929`)
 - Bug in :meth:`DataFrame.melt` where an exception was raised if ``var_name`` was not a string (:issue:`55948`)
 - Bug in :meth:`DataFrame.melt` where it would not preserve the datetime (:issue:`55254`)
diff --git a/pandas/core/reshape/merge.py b/pandas/core/reshape/merge.py
index f07c4fb8f7..c43c16cded 100644
--- a/pandas/core/reshape/merge.py
+++ b/pandas/core/reshape/merge.py
@@ -1986,7 +1986,12 @@ class _AsOfMerge(_OrderedMerge):
         else:
             ro_dtype = self.right.index.dtype
 
-        if is_object_dtype(lo_dtype) or is_object_dtype(ro_dtype):
+        if (
+            is_object_dtype(lo_dtype)
+            or is_object_dtype(ro_dtype)
+            or is_string_dtype(lo_dtype)
+            or is_string_dtype(ro_dtype)
+        ):
             raise MergeError(
                 f"Incompatible merge dtype, {repr(ro_dtype)} and "
                 f"{repr(lo_dtype)}, both sides must have numeric dtype"
diff --git a/pandas/io/excel/_base.py b/pandas/io/excel/_base.py
index 6d66830ab1..2884294377 100644
--- a/pandas/io/excel/_base.py
+++ b/pandas/io/excel/_base.py
@@ -86,7 +86,7 @@ if TYPE_CHECKING:
     )
 _read_excel_doc = (
     """
-Read an Excel file into a pandas DataFrame.
+Read an Excel file into a ``pandas`` ``DataFrame``.
 
 Supports `xls`, `xlsx`, `xlsm`, `xlsb`, `odf`, `ods` and `odt` file extensions
 read from a local filesystem or URL. Supports an option to read
@@ -112,7 +112,7 @@ sheet_name : str, int, list, or None, default 0
     Strings are used for sheet names. Integers are used in zero-indexed
     sheet positions (chart sheets do not count as a sheet position).
     Lists of strings/integers are used to request multiple sheets.
-    Specify None to get all worksheets.
+    Specify ``None`` to get all worksheets.
 
     Available cases:
 
@@ -121,7 +121,7 @@ sheet_name : str, int, list, or None, default 0
     * ``"Sheet1"``: Load sheet with name "Sheet1"
     * ``[0, 1, "Sheet5"]``: Load first, second and sheet named "Sheet5"
       as a dict of `DataFrame`
-    * None: All worksheets.
+    * ``None``: All worksheets.
 
 header : int, list of int, default 0
     Row (0-indexed) to use for the column labels of the parsed
@@ -155,21 +155,21 @@ usecols : str, list-like, or callable, default None
     Returns a subset of the columns according to behavior above.
 dtype : Type name or dict of column -> type, default None
     Data type for data or columns. E.g. {{'a': np.float64, 'b': np.int32}}
-    Use `object` to preserve data as stored in Excel and not interpret dtype,
-    which will necessarily result in `object` dtype.
+    Use ``object`` to preserve data as stored in Excel and not interpret dtype,
+    which will necessarily result in ``object`` dtype.
     If converters are specified, they will be applied INSTEAD
     of dtype conversion.
-    If you use `None`, it will infer the dtype of each column based on the data.
+    If you use ``None``, it will infer the dtype of each column based on the data.
 engine : str, default None
     If io is not a buffer or path, this must be set to identify io.
     Supported engines: "xlrd", "openpyxl", "odf", "pyxlsb", "calamine".
     Engine compatibility :
 
-    - "xlrd" supports old-style Excel files (.xls).
-    - "openpyxl" supports newer Excel file formats.
-    - "odf" supports OpenDocument file formats (.odf, .ods, .odt).
-    - "pyxlsb" supports Binary Excel files.
-    - "calamine" supports Excel (.xls, .xlsx, .xlsm, .xlsb)
+    - ``xlr`` supports old-style Excel files (.xls).
+    - ``openpyxl`` supports newer Excel file formats.
+    - ``odf`` supports OpenDocument file formats (.odf, .ods, .odt).
+    - ``pyxlsb`` supports Binary Excel files.
+    - ``calamine`` supports Excel (.xls, .xlsx, .xlsm, .xlsb)
       and OpenDocument (.ods) file formats.
 
     .. versionchanged:: 1.2.0
@@ -215,34 +215,34 @@ na_values : scalar, str, list-like, or dict, default None
     + """'.
 keep_default_na : bool, default True
     Whether or not to include the default NaN values when parsing the data.
-    Depending on whether `na_values` is passed in, the behavior is as follows:
+    Depending on whether ``na_values`` is passed in, the behavior is as follows:
 
-    * If `keep_default_na` is True, and `na_values` are specified, `na_values`
-      is appended to the default NaN values used for parsing.
-    * If `keep_default_na` is True, and `na_values` are not specified, only
+    * If ``keep_default_na`` is True, and ``na_values`` are specified,
+      ``na_values`` is appended to the default NaN values used for parsing.
+    * If ``keep_default_na`` is True, and ``na_values`` are not specified, only
       the default NaN values are used for parsing.
-    * If `keep_default_na` is False, and `na_values` are specified, only
-      the NaN values specified `na_values` are used for parsing.
-    * If `keep_default_na` is False, and `na_values` are not specified, no
+    * If ``keep_default_na`` is False, and ``na_values`` are specified, only
+      the NaN values specified ``na_values`` are used for parsing.
+    * If ``keep_default_na`` is False, and ``na_values`` are not specified, no
       strings will be parsed as NaN.
 
-    Note that if `na_filter` is passed in as False, the `keep_default_na` and
-    `na_values` parameters will be ignored.
+    Note that if `na_filter` is passed in as False, the ``keep_default_na`` and
+    ``na_values`` parameters will be ignored.
 na_filter : bool, default True
     Detect missing value markers (empty strings and the value of na_values). In
-    data without any NAs, passing na_filter=False can improve the performance
-    of reading a large file.
+    data without any NAs, passing ``na_filter=False`` can improve the
+    performance of reading a large file.
 verbose : bool, default False
     Indicate number of NA values placed in non-numeric columns.
 parse_dates : bool, list-like, or dict, default False
     The behavior is as follows:
 
-    * bool. If True -> try parsing the index.
-    * list of int or names. e.g. If [1, 2, 3] -> try parsing columns 1, 2, 3
+    * ``bool``. If True -> try parsing the index.
+    * ``list`` of int or names. e.g. If [1, 2, 3] -> try parsing columns 1, 2, 3
       each as a separate date column.
-    * list of lists. e.g.  If [[1, 3]] -> combine columns 1 and 3 and parse as
+    * ``list`` of lists. e.g.  If [[1, 3]] -> combine columns 1 and 3 and parse as
       a single date column.
-    * dict, e.g. {{'foo' : [1, 3]}} -> parse columns 1, 3 as date and call
+    * ``dict``, e.g. {{'foo' : [1, 3]}} -> parse columns 1, 3 as date and call
       result 'foo'
 
     If a column or index contains an unparsable date, the entire column or
@@ -372,7 +372,8 @@ as strings or lists of strings!
 1       NaN      2
 2  #Comment      3
 
-Comment lines in the excel input file can be skipped using the `comment` kwarg
+Comment lines in the excel input file can be skipped using the
+``comment`` kwarg.
 
 >>> pd.read_excel('tmp.xlsx', index_col=0, comment='#')  # doctest: +SKIP
       Name  Value


Test patch:
diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
index ffcd2ae32c..88d705dbd9 100644
--- a/.github/workflows/unit-tests.yml
+++ b/.github/workflows/unit-tests.yml
@@ -354,7 +354,7 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Python Dev Version
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.12-dev'
 
diff --git a/pandas/tests/frame/methods/test_to_dict_of_blocks.py b/pandas/tests/frame/methods/test_to_dict_of_blocks.py
index f7d9dc914a..f64cfd5fe6 100644
--- a/pandas/tests/frame/methods/test_to_dict_of_blocks.py
+++ b/pandas/tests/frame/methods/test_to_dict_of_blocks.py
@@ -35,9 +35,7 @@ class TestToDictOfBlocks:
             assert _last_df is not None and not _last_df[column].equals(df[column])
 
 
-def test_to_dict_of_blocks_item_cache(request, using_copy_on_write, warn_copy_on_write):
-    if using_copy_on_write:
-        request.applymarker(pytest.mark.xfail(reason="CoW - not yet implemented"))
+def test_to_dict_of_blocks_item_cache(using_copy_on_write, warn_copy_on_write):
     # Calling to_dict_of_blocks should not poison item_cache
     df = DataFrame({"a": [1, 2, 3, 4], "b": ["a", "b", "c", "d"]})
     df["c"] = NumpyExtensionArray(np.array([1, 2, None, 3], dtype=object))
@@ -49,10 +47,8 @@ def test_to_dict_of_blocks_item_cache(request, using_copy_on_write, warn_copy_on
     df._to_dict_of_blocks()
 
     if using_copy_on_write:
-        # TODO(CoW) we should disallow this, so `df` doesn't get updated,
-        # this currently still updates df, so this test fails
-        ser.values[0] = "foo"
-        assert df.loc[0, "b"] == "a"
+        with pytest.raises(ValueError, match="read-only"):
+            ser.values[0] = "foo"
     elif warn_copy_on_write:
         ser.values[0] = "foo"
         assert df.loc[0, "b"] == "foo"
diff --git a/pandas/tests/reshape/concat/test_append_common.py b/pandas/tests/reshape/concat/test_append_common.py
index ab20e8c8f6..31c3ef3176 100644
--- a/pandas/tests/reshape/concat/test_append_common.py
+++ b/pandas/tests/reshape/concat/test_append_common.py
@@ -57,10 +57,12 @@ class TestConcatAppendCommon:
     Test common dtype coercion rules between concat and append.
     """
 
-    def test_dtypes(self, item, index_or_series):
+    def test_dtypes(self, item, index_or_series, using_infer_string):
         # to confirm test case covers intended dtypes
         typ, vals = item
         obj = index_or_series(vals)
+        if typ == "object" and using_infer_string:
+            typ = "string"
         if isinstance(obj, Index):
             assert obj.dtype == typ
         elif isinstance(obj, Series):
diff --git a/pandas/tests/reshape/concat/test_categorical.py b/pandas/tests/reshape/concat/test_categorical.py
index 7acd0ff4f4..bbaaf0abec 100644
--- a/pandas/tests/reshape/concat/test_categorical.py
+++ b/pandas/tests/reshape/concat/test_categorical.py
@@ -51,7 +51,7 @@ class TestCategoricalConcat:
         exp["h"] = exp["h"].astype(df2["h"].dtype)
         tm.assert_frame_equal(res, exp)
 
-    def test_categorical_concat_dtypes(self):
+    def test_categorical_concat_dtypes(self, using_infer_string):
         # GH8143
         index = ["cat", "obj", "num"]
         cat = Categorical(["a", "b", "c"])
@@ -59,7 +59,9 @@ class TestCategoricalConcat:
         num = Series([1, 2, 3])
         df = pd.concat([Series(cat), obj, num], axis=1, keys=index)
 
-        result = df.dtypes == "object"
+        result = df.dtypes == (
+            object if not using_infer_string else "string[pyarrow_numpy]"
+        )
         expected = Series([False, True, False], index=index)
         tm.assert_series_equal(result, expected)
 
diff --git a/pandas/tests/reshape/concat/test_empty.py b/pandas/tests/reshape/concat/test_empty.py
index c80f3244dc..30ef0a9341 100644
--- a/pandas/tests/reshape/concat/test_empty.py
+++ b/pandas/tests/reshape/concat/test_empty.py
@@ -13,7 +13,7 @@ import pandas._testing as tm
 
 
 class TestEmptyConcat:
-    def test_handle_empty_objects(self, sort):
+    def test_handle_empty_objects(self, sort, using_infer_string):
         df = DataFrame(
             np.random.default_rng(2).standard_normal((10, 4)), columns=list("abcd")
         )
@@ -26,7 +26,9 @@ class TestEmptyConcat:
         concatted = concat(frames, axis=0, sort=sort)
 
         expected = df.reindex(columns=["a", "b", "c", "d", "foo"])
-        expected["foo"] = expected["foo"].astype("O")
+        expected["foo"] = expected["foo"].astype(
+            object if not using_infer_string else "string[pyarrow_numpy]"
+        )
         expected.loc[0:4, "foo"] = "bar"
 
         tm.assert_frame_equal(concatted, expected)
@@ -275,14 +277,14 @@ class TestEmptyConcat:
         expected = DataFrame(columns=["a", "b"])
         tm.assert_frame_equal(result, expected)
 
-    def test_concat_empty_dataframe_different_dtypes(self):
+    def test_concat_empty_dataframe_different_dtypes(self, using_infer_string):
         # 39037
         df1 = DataFrame({"a": [1, 2, 3], "b": ["a", "b", "c"]})
         df2 = DataFrame({"a": [1, 2, 3]})
 
         result = concat([df1[:0], df2[:0]])
         assert result["a"].dtype == np.int64
-        assert result["b"].dtype == np.object_
+        assert result["b"].dtype == np.object_ if not using_infer_string else "string"
 
     def test_concat_to_empty_ea(self):
         """48510 `concat` to an empty EA should maintain type EA dtype."""
diff --git a/pandas/tests/reshape/concat/test_index.py b/pandas/tests/reshape/concat/test_index.py
index f835bb953c..52bb9fa0f1 100644
--- a/pandas/tests/reshape/concat/test_index.py
+++ b/pandas/tests/reshape/concat/test_index.py
@@ -447,12 +447,14 @@ class TestMultiIndexConcat:
         )
         tm.assert_frame_equal(result, expected)
 
-    def test_concat_axis_1_sort_false_rangeindex(self):
+    def test_concat_axis_1_sort_false_rangeindex(self, using_infer_string):
         # GH 46675
         s1 = Series(["a", "b", "c"])
         s2 = Series(["a", "b"])
         s3 = Series(["a", "b", "c", "d"])
-        s4 = Series([], dtype=object)
+        s4 = Series(
+            [], dtype=object if not using_infer_string else "string[pyarrow_numpy]"
+        )
         result = concat(
             [s1, s2, s3, s4], sort=False, join="outer", ignore_index=False, axis=1
         )
@@ -463,7 +465,7 @@ class TestMultiIndexConcat:
                 ["c", np.nan] * 2,
                 [np.nan] * 2 + ["d"] + [np.nan],
             ],
-            dtype=object,
+            dtype=object if not using_infer_string else "string[pyarrow_numpy]",
         )
         tm.assert_frame_equal(
             result, expected, check_index_type=True, check_column_type=True
diff --git a/pandas/tests/reshape/merge/test_merge_asof.py b/pandas/tests/reshape/merge/test_merge_asof.py
index 6d0a405430..f6278e5e2f 100644
--- a/pandas/tests/reshape/merge/test_merge_asof.py
+++ b/pandas/tests/reshape/merge/test_merge_asof.py
@@ -11,6 +11,7 @@ from pandas import (
     Index,
     Timedelta,
     merge_asof,
+    option_context,
     to_datetime,
 )
 import pandas._testing as tm
@@ -3372,6 +3373,9 @@ class TestAsOfMerge:
         tm.assert_frame_equal(result, expected)
 
 
+@pytest.mark.parametrize(
+    "infer_string", [False, pytest.param(True, marks=td.skip_if_no("pyarrow"))]
+)
 @pytest.mark.parametrize(
     "kwargs", [{"on": "x"}, {"left_index": True, "right_index": True}]
 )
@@ -3379,15 +3383,16 @@ class TestAsOfMerge:
     "data",
     [["2019-06-01 00:09:12", "2019-06-01 00:10:29"], [1.0, "2019-06-01 00:10:29"]],
 )
-def test_merge_asof_non_numerical_dtype(kwargs, data):
+def test_merge_asof_non_numerical_dtype(kwargs, data, infer_string):
     # GH#29130
-    left = pd.DataFrame({"x": data}, index=data)
-    right = pd.DataFrame({"x": data}, index=data)
-    with pytest.raises(
-        MergeError,
-        match=r"Incompatible merge dtype, .*, both sides must have numeric dtype",
-    ):
-        merge_asof(left, right, **kwargs)
+    with option_context("future.infer_string", infer_string):
+        left = pd.DataFrame({"x": data}, index=data)
+        right = pd.DataFrame({"x": data}, index=data)
+        with pytest.raises(
+            MergeError,
+            match=r"Incompatible merge dtype, .*, both sides must have numeric dtype",
+        ):
+            merge_asof(left, right, **kwargs)
 
 
 def test_merge_asof_non_numerical_dtype_object():