Repository: pandas-dev/pandas
Pull request: BUG: Make nullable booleans numeric
- [x] closes #34051
- [x] tests added / passed
- [x] passes `black pandas`
- [x] passes `git diff upstream/master -u -- "*.py" | flake8 --diff`
- [x] whatsnew entry

Also a decent speedup since this is now going through Cython space:
```python
df = pd.DataFrame({"a": [1] * 50_000 + [2] * 50_000, "b": pd.array([True] * 100_000)})

# master
%timeit df.groupby("a").sum()                                                                                                                                                                  
# 5.15 ms ± 18 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)

# branch
%timeit df.groupby("a").sum()                                                                                                                                                                  
# 1.94 ms ± 10.8 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)
```

Problem statement:
BUG: DataFrameGroupBy.sum ignores min_count for boolean data type
- [x] I have checked that this issue has not already been reported.
- [x] I have confirmed this bug exists on the latest version of pandas.
- [x] (optional) I have confirmed this bug exists on the master branch of pandas.

---

Behavior is from master:

```python
import pandas as pd

df = pd.DataFrame({"a": [1, 2], "b": pd.array([True, True])})
df.groupby("a").sum(min_count=2)
```
gives
```python
      b
a      
1  True
2  True
```
but expected output is
```python
      b
a      
1  <NA>
2  <NA>
```

It looks to me like there's an attempt to compute a Cythonized result which fails, after which point the min_count argument is forgotten.

Hints from issue discussion:

Commit message: Merge remote-tracking branch 'upstream/master' into boolean-is-numeric

Patch to review:
diff --git a/doc/source/whatsnew/v1.1.0.rst b/doc/source/whatsnew/v1.1.0.rst
index 84ad478226175..e6cb847406473 100644
--- a/doc/source/whatsnew/v1.1.0.rst
+++ b/doc/source/whatsnew/v1.1.0.rst
@@ -800,6 +800,7 @@ ExtensionArray
 - Fixed bug that caused :meth:`Series.__repr__()` to crash for extension types whose elements are multidimensional arrays (:issue:`33770`).
 - Fixed bug where :meth:`Series.update` would raise a ``ValueError`` for ``ExtensionArray`` dtypes with missing values (:issue:`33980`)
 - Fixed bug where :meth:`StringArray.memory_usage` was not implemented (:issue:`33963`)
+- Fixed bug where :meth:`DataFrameGroupBy` would ignore the ``min_count`` argument for aggregations on nullable boolean dtypes (:issue:`34051`)
 - Fixed bug that `DataFrame(columns=.., dtype='string')` would fail (:issue:`27953`, :issue:`33623`)
 
 
diff --git a/pandas/core/arrays/boolean.py b/pandas/core/arrays/boolean.py
index b1d318d4d9678..95bc334905891 100644
--- a/pandas/core/arrays/boolean.py
+++ b/pandas/core/arrays/boolean.py
@@ -93,6 +93,10 @@ def __repr__(self) -> str:
     def _is_boolean(self) -> bool:
         return True
 
+    @property
+    def _is_numeric(self) -> bool:
+        return True
+
     def __from_arrow__(
         self, array: Union["pyarrow.Array", "pyarrow.ChunkedArray"]
     ) -> "BooleanArray":
diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py
index 9865a7d28542d..39c54b440cd4d 100644
--- a/pandas/core/dtypes/cast.py
+++ b/pandas/core/dtypes/cast.py
@@ -73,6 +73,8 @@
 from pandas.core.dtypes.inference import is_list_like
 from pandas.core.dtypes.missing import isna, notna
 
+import pandas as pd
+
 if TYPE_CHECKING:
     from pandas import Series
     from pandas.core.arrays import ExtensionArray  # noqa: F401
@@ -312,12 +314,11 @@ def maybe_cast_result_dtype(dtype: DtypeObj, how: str) -> DtypeObj:
     DtypeObj
         The desired dtype of the result.
     """
-    d = {
-        (np.dtype(np.bool), "add"): np.dtype(np.int64),
-        (np.dtype(np.bool), "cumsum"): np.dtype(np.int64),
-        (np.dtype(np.bool), "sum"): np.dtype(np.int64),
-    }
-    return d.get((dtype, how), dtype)
+    if how in ["add", "cumsum", "sum"] and (dtype == np.dtype(np.bool)):
+        return np.dtype(np.int64)
+    if how in ["add", "cumsum", "sum"] and isinstance(dtype, pd.BooleanDtype):
+        return pd.Int64Dtype()
+    return dtype
 
 
 def maybe_cast_to_extension_array(cls: Type["ExtensionArray"], obj, dtype=None):
diff --git a/pandas/core/groupby/ops.py b/pandas/core/groupby/ops.py
index d7ec5c63e16b9..4a0fbf071f7f9 100644
--- a/pandas/core/groupby/ops.py
+++ b/pandas/core/groupby/ops.py
@@ -485,7 +485,7 @@ def _cython_operation(
             values = values.view("int64")
             is_numeric = True
         elif is_bool_dtype(values.dtype):
-            values = ensure_float64(values)
+            values = ensure_int_or_float(values)
         elif is_integer_dtype(values):
             # we use iNaT for the missing value on ints
             # so pre-convert to guard this condition
diff --git a/pandas/tests/extension/test_boolean.py b/pandas/tests/extension/test_boolean.py
index 0b0bbd3a6dc48..725067951eeef 100644
--- a/pandas/tests/extension/test_boolean.py
+++ b/pandas/tests/extension/test_boolean.py
@@ -326,6 +326,23 @@ def test_in_numeric_groupby(self, data_for_grouping):
 
         tm.assert_index_equal(result, expected)
 
+    @pytest.mark.parametrize("min_count", [0, 10])
+    def test_groupby_sum_mincount(self, data_for_grouping, min_count):
+        df = pd.DataFrame({"A": [1, 1, 2, 2, 3, 3, 1], "B": data_for_grouping})
+        result = df.groupby("A").sum(min_count=min_count)
+        if min_count == 0:
+            expected = pd.DataFrame(
+                {"B": pd.array([3, 0, 0], dtype="Int64")},
+                index=pd.Index([1, 2, 3], name="A"),
+            )
+            tm.assert_frame_equal(result, expected)
+        else:
+            expected = pd.DataFrame(
+                {"B": pd.array([pd.NA] * 3, dtype="Int64")},
+                index=pd.Index([1, 2, 3], name="A"),
+            )
+            tm.assert_frame_equal(result, expected)
+
 
 class TestNumericReduce(base.BaseNumericReduceTests):
     def check_reduce(self, s, op_name, skipna):


Code review comments:
diff_hunk: @@ -312,12 +314,11 @@ def maybe_cast_result_dtype(dtype: DtypeObj, how: str) -> DtypeObj:
     DtypeObj
         The desired dtype of the result.
     """
-    d = {
-        (np.dtype(np.bool), "add"): np.dtype(np.int64),
-        (np.dtype(np.bool), "cumsum"): np.dtype(np.int64),
-        (np.dtype(np.bool), "sum"): np.dtype(np.int64),
-    }
-    return d.get((dtype, how), dtype)
+    if how in ["add", "cumsum", "sum"] and (dtype == np.dtype(np.bool)):
+        return np.dtype(np.int64)
+    if how in ["add", "cumsum", "sum"] and isinstance(dtype, pd.BooleanDtype):
path: pandas/core/dtypes/cast.py
text: @user1:
can be elif

diff_hunk: @@ -312,12 +314,11 @@ def maybe_cast_result_dtype(dtype: DtypeObj, how: str) -> DtypeObj:
     DtypeObj
         The desired dtype of the result.
     """
-    d = {
-        (np.dtype(np.bool), "add"): np.dtype(np.int64),
-        (np.dtype(np.bool), "cumsum"): np.dtype(np.int64),
-        (np.dtype(np.bool), "sum"): np.dtype(np.int64),
-    }
-    return d.get((dtype, how), dtype)
+    if how in ["add", "cumsum", "sum"] and (dtype == np.dtype(np.bool)):
+        return np.dtype(np.int64)
+    if how in ["add", "cumsum", "sum"] and isinstance(dtype, pd.BooleanDtype):
path: pandas/core/dtypes/cast.py
text: @user1:
dont use ``pd`` import *anywhere*, rather prefer to import directly.

@author:
I'm hitting some circular import issues when I try to import these directly, is it okay to import from within the function itself?

diff_hunk: @@ -312,12 +314,11 @@ def maybe_cast_result_dtype(dtype: DtypeObj, how: str) -> DtypeObj:
     DtypeObj
         The desired dtype of the result.
     """
-    d = {
-        (np.dtype(np.bool), "add"): np.dtype(np.int64),
-        (np.dtype(np.bool), "cumsum"): np.dtype(np.int64),
-        (np.dtype(np.bool), "sum"): np.dtype(np.int64),
-    }
-    return d.get((dtype, how), dtype)
+    if how in ["add", "cumsum", "sum"] and (dtype == np.dtype(np.bool)):
path: pandas/core/dtypes/cast.py
text: @user1:
use
```issubclass(dtype, np.bool_)```

@author:
This ends up raising in a test:
```python
E       TypeError: issubclass() arg 1 must be a class
```
I think for np.dtype("float64") as the input

@user1:
use type(dtype)

@user1:
or i guess its ok as is

@user1:
though maybe

```np.dtype(dtype) == "bool"``` is better

@user1:
depends on what we allow dtype at this point

diff_hunk: @@ -312,12 +314,11 @@ def maybe_cast_result_dtype(dtype: DtypeObj, how: str) -> DtypeObj:
     DtypeObj
         The desired dtype of the result.
     """
-    d = {
-        (np.dtype(np.bool), "add"): np.dtype(np.int64),
-        (np.dtype(np.bool), "cumsum"): np.dtype(np.int64),
-        (np.dtype(np.bool), "sum"): np.dtype(np.int64),
-    }
-    return d.get((dtype, how), dtype)
+    if how in ["add", "cumsum", "sum"] and (dtype == np.dtype(np.bool)):
+        return np.dtype(np.int64)
+    if how in ["add", "cumsum", "sum"] and isinstance(dtype, pd.BooleanDtype):
+        return pd.Int64Dtype()
path: pandas/core/dtypes/cast.py
text: @user1:
don't use pd

@user1:
you can either import (best) or return a string "Int64"