Repository: pandas-dev/pandas
Pull request: BUG: DataFrameGroupBy.__getitem__ fails to propagate dropna
- [x] closes #35014 
- [x] tests added / passed
- [x] passes `black pandas`
- [x] passes `git diff upstream/master -u -- "*.py" | flake8 --diff`
- [x] whatsnew entry

Problem statement:
BUG: DataFrameGroupBy.__getitem__ fails to propagate dropna
#### Code Sample, a copy-pastable example

```pytb
In [1]: import pandas as pd
In [2]: df = pd.DataFrame({"A": [0, 0, 1, None], "B": [1, 2, 3, None]})
In [3]: gb = df.groupby("A", dropna=False)
In [6]: gb['B'].transform(len)
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-6-3bae7d67a46f> in <module>
----> 1 gb['B'].transform(len)

~/sandbox/pandas/pandas/core/groupby/generic.py in transform(self, func, engine, engine_kwargs, *args, **kwargs)
    471         if not isinstance(func, str):
    472             return self._transform_general(
--> 473                 func, *args, engine=engine, engine_kwargs=engine_kwargs, **kwargs
    474             )
    475

~/sandbox/pandas/pandas/core/groupby/generic.py in _transform_general(self, func, engine, engine_kwargs, *args, **kwargs)
    537
    538         result.name = self._selected_obj.name
--> 539         result.index = self._selected_obj.index
    540         return result
    541

~/sandbox/pandas/pandas/core/generic.py in __setattr__(self, name, value)
   5141         try:
   5142             object.__getattribute__(self, name)
-> 5143             return object.__setattr__(self, name, value)
   5144         except AttributeError:
   5145             pass

~/sandbox/pandas/pandas/_libs/properties.pyx in pandas._libs.properties.AxisProperty.__set__()
     64
     65     def __set__(self, obj, value):
---> 66         obj._set_axis(self.axis, value)

~/sandbox/pandas/pandas/core/series.py in _set_axis(self, axis, labels, fastpath)
    422         if not fastpath:
    423             # The ensure_index call above ensures we have an Index object
--> 424             self._mgr.set_axis(axis, labels)
    425
    426     # ndarray compatibility

~/sandbox/pandas/pandas/core/internals/managers.py in set_axis(self, axis, new_labels)
    213         if new_len != old_len:
    214             raise ValueError(
--> 215                 f"Length mismatch: Expected axis has {old_len} elements, new "
    216                 f"values have {new_len} elements"
    217             )

ValueError: Length mismatch: Expected axis has 3 elements, new values have 4 elements
```

#### Problem description

Compare that with the following

```python
In [4]: gb.transform(len)
Out[4]:
   B
0  2
1  2
2  1
3  1

In [5]: gb[['B']].transform(len)
Out[5]:
   B
0  2
1  2
2  1
3  1
```

So it's just when slicing down to a SeriesGroupBy object.

#### Expected Output

A series:

```python
Out[5]:
0  2
1  2
2  1
3  1
```

Hints from issue discussion:
take
@TomAugspurger I think that the problem in `SeriesGroupBy.transform` comes down to L387-388 in `pandas/core/groupby/grouper.py`:
https://github.com/pandas-dev/pandas/blob/b687cd4d9e520666a956a60849568a98dd00c672/pandas/core/groupby/grouper.py#L387
For `gb['B']` if we print out `values` `NaN` is getting getting dropped (and this then propagates along to where the original issue came up):
```
self.grouper = [ 0.  0.  1. nan]

values = {'_dtype': CategoricalDtype(categories=[0.0, 1.0], ordered=False), '_codes': array([ 0,  0,  1, -1], dtype=int8)}
```
Since what we need from `indices` is a dict of values and indices where they occur a quick solution could be to do that on the fly in `Grouping.indices`. Would that work or would we want to do something else?

Commit message: Merge remote-tracking branch 'upstream/master' into groupby-getitem-dropna-doesnt-propagate

Patch to review:
diff --git a/doc/source/whatsnew/v1.2.0.rst b/doc/source/whatsnew/v1.2.0.rst
index 260b92b5989c1..df010c3776f7c 100644
--- a/doc/source/whatsnew/v1.2.0.rst
+++ b/doc/source/whatsnew/v1.2.0.rst
@@ -128,7 +128,7 @@ Indexing
 Missing
 ^^^^^^^
 
--
+- Bug in :meth:`SeriesGroupBy.transform` now correctly handles missing values for `dropna=False` (:issue:`35014`)
 -
 
 MultiIndex
diff --git a/pandas/core/groupby/generic.py b/pandas/core/groupby/generic.py
index 740463f0cf356..bada7060fb515 100644
--- a/pandas/core/groupby/generic.py
+++ b/pandas/core/groupby/generic.py
@@ -39,7 +39,6 @@
     maybe_cast_result_dtype,
     maybe_convert_objects,
     maybe_downcast_numeric,
-    maybe_downcast_to_dtype,
 )
 from pandas.core.dtypes.common import (
     ensure_int64,
@@ -535,24 +534,25 @@ def _transform_general(
             if isinstance(res, (ABCDataFrame, ABCSeries)):
                 res = res._values
 
-            indexer = self._get_index(name)
-            ser = klass(res, indexer)
-            results.append(ser)
+            results.append(klass(res, index=group.index))
 
         # check for empty "results" to avoid concat ValueError
         if results:
             from pandas.core.reshape.concat import concat
 
-            result = concat(results).sort_index()
+            concatenated = concat(results)
+            result = self._set_result_index_ordered(concatenated)
         else:
             result = self.obj._constructor(dtype=np.float64)
-
         # we will only try to coerce the result type if
         # we have a numeric dtype, as these are *always* user-defined funcs
         # the cython take a different path (and casting)
-        dtype = self._selected_obj.dtype
-        if is_numeric_dtype(dtype):
-            result = maybe_downcast_to_dtype(result, dtype)
+        if is_numeric_dtype(result.dtype):
+            common_dtype = np.find_common_type(
+                [self._selected_obj.dtype, result.dtype], []
+            )
+            if common_dtype is result.dtype:
+                result = maybe_downcast_numeric(result, self._selected_obj.dtype)
 
         result.name = self._selected_obj.name
         result.index = self._selected_obj.index
diff --git a/pandas/tests/groupby/test_groupby_dropna.py b/pandas/tests/groupby/test_groupby_dropna.py
index 1a525d306e9f5..3f158e99fab31 100644
--- a/pandas/tests/groupby/test_groupby_dropna.py
+++ b/pandas/tests/groupby/test_groupby_dropna.py
@@ -162,6 +162,27 @@ def test_groupby_dropna_series_by(dropna, expected):
     tm.assert_series_equal(result, expected)
 
 
+def test_slice_groupby_then_transform():
+    # GH35014
+
+    df = pd.DataFrame({"A": [0, 0, 1, None], "B": [1, 2, 3, None]})
+    gb = df.groupby("A", dropna=False)
+
+    res = gb.transform(len)
+    expected = pd.DataFrame({"B": [2, 2, 1, 1]})
+    tm.assert_frame_equal(res, expected)
+
+    gb_slice = gb[["B"]]
+    res = gb_slice.transform(len)
+    expected = pd.DataFrame({"B": [2, 2, 1, 1]})
+    tm.assert_frame_equal(res, expected)
+
+    gb_slice = gb["B"]
+    res = gb["B"].transform(len)
+    expected = pd.Series(data=[2, 2, 1, 1], name="B")
+    tm.assert_series_equal(res, expected)
+
+
 @pytest.mark.parametrize(
     "dropna, tuples, outputs",
     [


Code review comments:
diff_hunk: @@ -535,24 +534,25 @@ def _transform_general(
             if isinstance(res, (ABCDataFrame, ABCSeries)):
                 res = res._values
 
-            indexer = self._get_index(name)
-            ser = klass(res, indexer)
-            results.append(ser)
+            results.append(klass(res, index=group.index))
 
         # check for empty "results" to avoid concat ValueError
         if results:
             from pandas.core.reshape.concat import concat
 
-            result = concat(results).sort_index()
+            concatenated = concat(results)
+            result = self._set_result_index_ordered(concatenated)
         else:
             result = self.obj._constructor(dtype=np.float64)
-
         # we will only try to coerce the result type if
         # we have a numeric dtype, as these are *always* user-defined funcs
         # the cython take a different path (and casting)
-        dtype = self._selected_obj.dtype
-        if is_numeric_dtype(dtype):
-            result = maybe_downcast_to_dtype(result, dtype)
+        if is_numeric_dtype(result.dtype):
+            common_dtype = np.find_common_type(
path: pandas/core/groupby/generic.py
text: @user1:
use pandas.core.dtypes.cast.find_common_dtype