Repository: pandas-dev/pandas
Pull request: BUG: ArrowExtensionArray.to_numpy avoid object dtype when na_value provided
- [x] closes #54808
- [x] [Tests added and passed](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#writing-tests) if fixing a bug or adding a new feature
- [x] All [code checks passed](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#pre-commit).
- [x] Added [type annotations](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#type-hints) to new arguments/methods/functions.


Problem statement:
BUG: avoid conversion to object dtype in to_numpy for Arrow dtype and with dtype-compatible fill value
Using the current main branch (2.1.0rc0+115.gca429943c7), if we convert an Arrow-backed boolean array to numpy, we get object dtype if there are missing values:

```python
# no missing values -> bool dtype
>>> ser = pd.Series([True, False, True], dtype=pd.ArrowDtype(pa.bool_()))
>>> ser.to_numpy()
array([ True, False,  True])

# missing values -> object dtype
>>> ser = pd.Series([True, False, None], dtype=pd.ArrowDtype(pa.bool_()))
>>> ser.to_numpy()
array([True, False, <NA>], dtype=object)
```

This follows pyarrow's behaviour of converting to a numpy array (and I think also makes sense, unless we might want to raise an error instead, but that's for another discussion). 

But when a user specifies an `na_value` to avoid this issue of missing values with numpy, we still return an object dtype:

```python
>>> ser.to_numpy(na_value=False)
array([True, False, False], dtype=object)
```

while if the specified `na_value` is valid for the default target bool dtype (as in the example above), we could perfectly well return a proper bool dtype array automatically (currently you would have to be explicit: `to_numpy(np.bool_, na_value=False)`).

Hints from issue discussion:
cc @lukemanley IIRC you spent some time optimizing `ArrowExtensionArray.to_numpy`

Commit message: refactor

Patch to review:
diff --git a/pandas/core/arrays/arrow/array.py b/pandas/core/arrays/arrow/array.py
index e815b8292b0cc..dfa7b6076d591 100644
--- a/pandas/core/arrays/arrow/array.py
+++ b/pandas/core/arrays/arrow/array.py
@@ -29,12 +29,12 @@
 from pandas.util._decorators import doc
 from pandas.util._validators import validate_fillna_kwargs
 
+from pandas.core.dtypes.cast import can_hold_element
 from pandas.core.dtypes.common import (
     is_array_like,
     is_bool_dtype,
     is_integer,
     is_list_like,
-    is_object_dtype,
     is_scalar,
 )
 from pandas.core.dtypes.dtypes import DatetimeTZDtype
@@ -1240,46 +1240,50 @@ def to_numpy(
     ) -> np.ndarray:
         if dtype is not None:
             dtype = np.dtype(dtype)
-        elif self._hasna:
-            dtype = np.dtype(object)
 
         if na_value is lib.no_default:
             na_value = self.dtype.na_value
 
         pa_type = self._pa_array.type
+        if not self._hasna or isna(na_value) or pa.types.is_null(pa_type):
+            data = self
+        else:
+            data = self.fillna(na_value)
+            copy = False
+
         if pa.types.is_timestamp(pa_type) or pa.types.is_duration(pa_type):
-            result = self._maybe_convert_datelike_array()
+            result = data._maybe_convert_datelike_array()
             if dtype is None or dtype.kind == "O":
                 result = result.to_numpy(dtype=object, na_value=na_value)
             else:
                 result = result.to_numpy(dtype=dtype)
-            return result
         elif pa.types.is_time(pa_type) or pa.types.is_date(pa_type):
             # convert to list of python datetime.time objects before
             # wrapping in ndarray
-            result = np.array(list(self), dtype=dtype)
-        elif is_object_dtype(dtype) and self._hasna:
-            result = np.empty(len(self), dtype=object)
-            mask = ~self.isna()
-            result[mask] = np.asarray(self[mask]._pa_array)
-        elif pa.types.is_null(self._pa_array.type):
-            fill_value = None if isna(na_value) else na_value
-            return np.full(len(self), fill_value=fill_value, dtype=dtype)
-        elif self._hasna:
-            data = self.fillna(na_value)
+            result = np.array(list(data), dtype=dtype)
+            if data._hasna:
+                result[data.isna()] = na_value
+        elif pa.types.is_null(pa_type):
+            if dtype is not None and dtype.kind != "O" and isna(na_value):
+                na_value = None
+            result = np.full(len(data), fill_value=na_value, dtype=dtype)
+        elif not data._hasna or (pa.types.is_floating(pa_type) and na_value is np.nan):
             result = data._pa_array.to_numpy()
-            if dtype is not None:
-                result = result.astype(dtype, copy=False)
-            return result
-        else:
-            result = self._pa_array.to_numpy()
             if dtype is not None:
                 result = result.astype(dtype, copy=False)
             if copy:
                 result = result.copy()
-            return result
-        if self._hasna:
-            result[self.isna()] = na_value
+        else:
+            if dtype is None:
+                empty = data._pa_array[:0].to_numpy()
+                if can_hold_element(empty, na_value):
+                    dtype = empty.dtype
+                else:
+                    dtype = np.object_
+            result = np.empty(len(data), dtype=dtype)
+            mask = data.isna()
+            result[mask] = na_value
+            result[~mask] = data[~mask]._pa_array.to_numpy()
         return result
 
     def unique(self) -> Self:
diff --git a/pandas/tests/arrays/string_/test_string.py b/pandas/tests/arrays/string_/test_string.py
index 24d8e43708b91..89cc31ec5ecc8 100644
--- a/pandas/tests/arrays/string_/test_string.py
+++ b/pandas/tests/arrays/string_/test_string.py
@@ -377,8 +377,16 @@ def test_astype_int(dtype):
     tm.assert_numpy_array_equal(result, expected)
 
     arr = pd.array(["1", pd.NA, "3"], dtype=dtype)
-    msg = r"int\(\) argument must be a string, a bytes-like object or a( real)? number"
-    with pytest.raises(TypeError, match=msg):
+    if dtype.storage == "pyarrow_numpy":
+        err = ValueError
+        msg = "cannot convert float NaN to integer"
+    else:
+        err = TypeError
+        msg = (
+            r"int\(\) argument must be a string, a bytes-like "
+            r"object or a( real)? number"
+        )
+    with pytest.raises(err, match=msg):
         arr.astype("int64")
 
 
diff --git a/pandas/tests/extension/test_arrow.py b/pandas/tests/extension/test_arrow.py
index 13d80329f4d51..5f1b16a44b8e9 100644
--- a/pandas/tests/extension/test_arrow.py
+++ b/pandas/tests/extension/test_arrow.py
@@ -1595,6 +1595,19 @@ def test_to_numpy_null_array_no_dtype():
     tm.assert_numpy_array_equal(result, expected)
 
 
+def test_to_numpy_without_dtype():
+    # GH 54808
+    arr = pd.array([True, pd.NA], dtype="boolean[pyarrow]")
+    result = arr.to_numpy(na_value=False)
+    expected = np.array([True, False], dtype=np.bool_)
+    tm.assert_numpy_array_equal(result, expected)
+
+    arr = pd.array([1.0, pd.NA], dtype="float32[pyarrow]")
+    result = arr.to_numpy(na_value=0.0)
+    expected = np.array([1.0, 0.0], dtype=np.float32)
+    tm.assert_numpy_array_equal(result, expected)
+
+
 def test_setitem_null_slice(data):
     # GH50248
     orig = data.copy()


Code review comments:
diff_hunk: @@ -1240,46 +1240,50 @@ def to_numpy(
     ) -> np.ndarray:
         if dtype is not None:
             dtype = np.dtype(dtype)
-        elif self._hasna:
-            dtype = np.dtype(object)
 
         if na_value is lib.no_default:
             na_value = self.dtype.na_value
 
         pa_type = self._pa_array.type
+        if not self._hasna or isna(na_value) or pa.types.is_null(pa_type):
+            data = self
+        else:
+            data = self.fillna(na_value)
+            copy = False
+
         if pa.types.is_timestamp(pa_type) or pa.types.is_duration(pa_type):
-            result = self._maybe_convert_datelike_array()
+            result = data._maybe_convert_datelike_array()
             if dtype is None or dtype.kind == "O":
                 result = result.to_numpy(dtype=object, na_value=na_value)
             else:
                 result = result.to_numpy(dtype=dtype)
-            return result
         elif pa.types.is_time(pa_type) or pa.types.is_date(pa_type):
             # convert to list of python datetime.time objects before
             # wrapping in ndarray
-            result = np.array(list(self), dtype=dtype)
-        elif is_object_dtype(dtype) and self._hasna:
-            result = np.empty(len(self), dtype=object)
-            mask = ~self.isna()
-            result[mask] = np.asarray(self[mask]._pa_array)
-        elif pa.types.is_null(self._pa_array.type):
-            fill_value = None if isna(na_value) else na_value
-            return np.full(len(self), fill_value=fill_value, dtype=dtype)
-        elif self._hasna:
-            data = self.fillna(na_value)
+            result = np.array(list(data), dtype=dtype)
+            if data._hasna:
+                result[data.isna()] = na_value
+        elif pa.types.is_null(pa_type):
+            if dtype is not None and dtype.kind != "O" and isna(na_value):
path: pandas/core/arrays/arrow/array.py
text: @user1:
Why do we need to skip object here?

@author:
removed it, not needed

diff_hunk: @@ -1240,46 +1240,50 @@ def to_numpy(
     ) -> np.ndarray:
         if dtype is not None:
             dtype = np.dtype(dtype)
-        elif self._hasna:
-            dtype = np.dtype(object)
 
         if na_value is lib.no_default:
             na_value = self.dtype.na_value
 
         pa_type = self._pa_array.type
+        if not self._hasna or isna(na_value) or pa.types.is_null(pa_type):
+            data = self
+        else:
+            data = self.fillna(na_value)
+            copy = False
+
         if pa.types.is_timestamp(pa_type) or pa.types.is_duration(pa_type):
-            result = self._maybe_convert_datelike_array()
+            result = data._maybe_convert_datelike_array()
             if dtype is None or dtype.kind == "O":
                 result = result.to_numpy(dtype=object, na_value=na_value)
             else:
                 result = result.to_numpy(dtype=dtype)
-            return result
         elif pa.types.is_time(pa_type) or pa.types.is_date(pa_type):
             # convert to list of python datetime.time objects before
             # wrapping in ndarray
-            result = np.array(list(self), dtype=dtype)
-        elif is_object_dtype(dtype) and self._hasna:
-            result = np.empty(len(self), dtype=object)
-            mask = ~self.isna()
-            result[mask] = np.asarray(self[mask]._pa_array)
-        elif pa.types.is_null(self._pa_array.type):
-            fill_value = None if isna(na_value) else na_value
-            return np.full(len(self), fill_value=fill_value, dtype=dtype)
-        elif self._hasna:
-            data = self.fillna(na_value)
+            result = np.array(list(data), dtype=dtype)
+            if data._hasna:
+                result[data.isna()] = na_value
+        elif pa.types.is_null(pa_type):
+            if dtype is not None and dtype.kind != "O" and isna(na_value):
+                na_value = None
+            result = np.full(len(data), fill_value=na_value, dtype=dtype)
+        elif not data._hasna or (pa.types.is_floating(pa_type) and na_value is np.nan):
             result = data._pa_array.to_numpy()
-            if dtype is not None:
-                result = result.astype(dtype, copy=False)
-            return result
-        else:
-            result = self._pa_array.to_numpy()
             if dtype is not None:
                 result = result.astype(dtype, copy=False)
             if copy:
                 result = result.copy()
-            return result
-        if self._hasna:
-            result[self.isna()] = na_value
+        else:
+            if dtype is None:
+                empty = data._pa_array[:0].to_numpy()
path: pandas/core/arrays/arrow/array.py
text: @user1:
I think `data.dtype.empty._pa_array` would be a "nicer" way to get an empty example to use in `can_hold_element`

@author:
yep, thanks. updated