Repository: geopandas/geopandas

Issue:
ENH: Add groupby's observed kwarg to dissolve
Using `dissolve` on a `GeoDataFrame` with categorical and non-categorical columns produces unwanted results.

Consider this `DataFrame`, which currently has no categorical columns.
```
>>> df = pd.DataFrame({"cat": ['a', 'a', 'b', 'b'],
                       "noncat": [1, 1, 1, 2],
                       "to_agg": [1, 2, 3, 4],
                       "geometry": ['POINT (0 0)', 'POINT (1 1)', 'POINT (2 2)', 'POINT (4 4)']})
```
If we turn this into a `GeoDataFrame` and dissolve it by the columns `cat` and `noncat`, we get the desired results.
```
>>> gdf = geopandas.GeoDataFrame(df, geometry=geopandas.array.from_wkt(df['geometry'].values))
>>> gdf.dissolve(['cat', 'noncat'])
                                                 geometry  to_agg
cat noncat
a   1       MULTIPOINT (0.00000 0.00000, 1.00000 1.00000)       1
b   1                             POINT (2.00000 2.00000)       3
    2                             POINT (4.00000 4.00000)       4
```

However, if we turn one of the groupby columns into a categorical, we do not get the desired results.
```
>>> df2 = df.astype({'cat': 'category'})
>>> gdf2 = geopandas.GeoDataFrame(df2, geometry=geopandas.array.from_wkt(df2['geometry'].values))
>>> gdf2.dissolve(['cat', 'noncat'])
                                                 geometry  to_agg
cat noncat
a   1       MULTIPOINT (0.00000 0.00000, 1.00000 1.00000)     1.0
    2                                                None     NaN
b   1                             POINT (2.00000 2.00000)     3.0
    2                             POINT (4.00000 4.00000)     4.0
```
We have an additional entry in the index for `cat='a'`, `noncat=2` which doesn't exist in the original data.

In [`pandas.groupby`](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.groupby.html), this behavior can be controlled by the `observed` parameter.
> **observed** _bool, default False_
This only applies if any of the groupers are Categoricals. If True: only show observed values for categorical groupers. If False: show all values for categorical groupers.

`observed` is not used in `GeoDataFrame.dissolve`'s calls to `groupby`, nor is it controllable with an arg. If we modify `GeoDataFrame.dissolve` to have an `observed=False` kwarg and pass `observed` into both `groupby` operations that it performs internally, then the desired result is obtained.

```python
def new_dissolve(self, by=None, aggfunc="first", as_index=True, observed=False):
    if by is None:
        by = np.zeros(len(self), dtype="int64")

    # Process non-spatial component
    data = self.drop(labels=self.geometry.name, axis=1)
    aggregated_data = data.groupby(by=by, observed=observed).agg(aggfunc)

    # Process spatial component
    def merge_geometries(block):
        merged_geom = block.unary_union
        return merged_geom

    g = self.groupby(by=by, group_keys=False, observed=observed)[self.geometry.name].agg(
        merge_geometries
    )

    # Aggregate
    aggregated_geometry = GeoDataFrame(g, geometry=self.geometry.name, crs=self.crs)
    # Recombine
    aggregated = aggregated_geometry.join(aggregated_data)

    # Reset if requested
    if not as_index:
        aggregated = aggregated.reset_index()

    return aggregated
```
```
>>> GeoDataFrame.new_dissolve = new_dissolve
>>> gdf2.new_dissolve(['cat', 'noncat'], observed=True)
                                                 geometry  to_agg
cat noncat
a   1       MULTIPOINT (0.00000 0.00000, 1.00000 1.00000)       1
b   1                             POINT (2.00000 2.00000)       3
    2                             POINT (4.00000 4.00000)       4
```

#### API breaking implications

None. We simply expose a kwarg on `dissolve` that already exists on the underlying `groupby` function. Giving it the default value `False` preserves the existing behavior.

#### Describe alternatives you've considered

In my application code, we have worked around this by converting all categorical columns into non-categorical columns (with the values being their category codes) before `groupby` and converting them back after.

```python
def categorical_workaround_pre_groupby(df):
    """Workaround (part a) for dissolve not exposing the 'observed' parameter for categorical"""
    categories_map = dict()
    for col in df:
        if isinstance(df[col].dtype, pd.CategoricalDtype):
            categories_map[col] = df[col].cat.categories
            df[col] = df[col].cat.codes
    return categories_map

def categorical_workaround_post_groupby(df, categories_map):
    """Workaround (part b) for dissolve not exposing the 'observed' parameter for categorical"""
    for col in categories_map:
        df[col] = pd.Categorical.from_codes(df[col], categories_map[col])
```

This is possible, it does what we need, but it is not ideal.



Discussion:

Resolving patch:
diff --git a/geopandas/geodataframe.py b/geopandas/geodataframe.py
--- a/geopandas/geodataframe.py
+++ b/geopandas/geodataframe.py
@@ -1312,7 +1312,16 @@ def __finalize__(self, other, method=None, **kwargs):
 
         return self
 
-    def dissolve(self, by=None, aggfunc="first", as_index=True):
+    def dissolve(
+        self,
+        by=None,
+        aggfunc="first",
+        as_index=True,
+        level=None,
+        sort=True,
+        observed=False,
+        dropna=True,
+    ):
         """
         Dissolve geometries within `groupby` into single observation.
         This is accomplished by applying the `unary_union` method
@@ -1331,6 +1340,33 @@ def dissolve(self, by=None, aggfunc="first", as_index=True):
             with each group. Passed to pandas `groupby.agg` method.
         as_index : boolean, default True
             If true, groupby columns become index of result.
+        level : int or str or sequence of int or sequence of str, default None
+            If the axis is a MultiIndex (hierarchical), group by a
+            particular level or levels.
+
+            .. versionadded:: 0.9.0
+        sort : bool, default True
+            Sort group keys. Get better performance by turning this off.
+            Note this does not influence the order of observations within
+            each group. Groupby preserves the order of rows within each group.
+
+            .. versionadded:: 0.9.0
+        observed : bool, default False
+            This only applies if any of the groupers are Categoricals.
+            If True: only show observed values for categorical groupers.
+            If False: show all values for categorical groupers.
+
+            .. versionadded:: 0.9.0
+        dropna : bool, default True
+            If True, and if group keys contain NA values, NA values
+            together with row/column will be dropped. If False, NA
+            values will also be treated as the key in groups.
+
+            This parameter is not supported for pandas < 1.1.0.
+            A warning will be emitted for earlier pandas versions
+            if a non-default value is given for this parameter.
+
+            .. versionadded:: 0.9.0
 
         Returns
         -------
@@ -1363,19 +1399,28 @@ def dissolve(self, by=None, aggfunc="first", as_index=True):
 
         """
 
-        if by is None:
+        if by is None and level is None:
             by = np.zeros(len(self), dtype="int64")
 
+        groupby_kwargs = dict(
+            by=by, level=level, sort=sort, observed=observed, dropna=dropna
+        )
+        if not compat.PANDAS_GE_11:
+            groupby_kwargs.pop("dropna")
+
+            if not dropna:  # If they passed a non-default dropna value
+                warnings.warn("dropna kwarg is not supported for pandas < 1.1.0")
+
         # Process non-spatial component
         data = self.drop(labels=self.geometry.name, axis=1)
-        aggregated_data = data.groupby(by=by).agg(aggfunc)
+        aggregated_data = data.groupby(**groupby_kwargs).agg(aggfunc)
 
         # Process spatial component
         def merge_geometries(block):
             merged_geom = block.unary_union
             return merged_geom
 
-        g = self.groupby(by=by, group_keys=False)[self.geometry.name].agg(
+        g = self.groupby(group_keys=False, **groupby_kwargs)[self.geometry.name].agg(
             merge_geometries
         )
 


Test patch:
diff --git a/geopandas/tests/test_dissolve.py b/geopandas/tests/test_dissolve.py
--- a/geopandas/tests/test_dissolve.py
+++ b/geopandas/tests/test_dissolve.py
@@ -3,6 +3,7 @@
 
 import geopandas
 from geopandas import GeoDataFrame, read_file
+from geopandas import _compat as compat
 
 from pandas.testing import assert_frame_equal
 import pytest
@@ -128,3 +129,175 @@ def test_dissolve_none_mean(nybb_polydf):
         crs=nybb_polydf.crs,
     )
     assert_frame_equal(expected, test, check_column_type=False)
+
+
+def test_dissolve_level():
+    gdf = geopandas.GeoDataFrame(
+        {
+            "a": [1, 1, 2, 2],
+            "b": [3, 4, 4, 4],
+            "c": [3, 4, 5, 6],
+            "geometry": geopandas.array.from_wkt(
+                ["POINT (0 0)", "POINT (1 1)", "POINT (2 2)", "POINT (3 3)"]
+            ),
+        }
+    ).set_index(["a", "b", "c"])
+
+    expected_a = geopandas.GeoDataFrame(
+        {
+            "a": [1, 2],
+            "geometry": geopandas.array.from_wkt(
+                ["MULTIPOINT (0 0, 1 1)", "MULTIPOINT (2 2, 3 3)"]
+            ),
+        }
+    ).set_index("a")
+    expected_b = geopandas.GeoDataFrame(
+        {
+            "b": [3, 4],
+            "geometry": geopandas.array.from_wkt(
+                ["POINT (0 0)", "MULTIPOINT (1 1, 2 2, 3 3)"]
+            ),
+        }
+    ).set_index("b")
+    expected_ab = geopandas.GeoDataFrame(
+        {
+            "a": [1, 1, 2],
+            "b": [3, 4, 4],
+            "geometry": geopandas.array.from_wkt(
+                ["POINT (0 0)", "POINT (1 1)", "MULTIPOINT (2 2, 3 3)"]
+            ),
+        }
+    ).set_index(["a", "b"])
+
+    assert_frame_equal(expected_a, gdf.dissolve(level=0))
+    assert_frame_equal(expected_a, gdf.dissolve(level="a"))
+    assert_frame_equal(expected_b, gdf.dissolve(level=1))
+    assert_frame_equal(expected_b, gdf.dissolve(level="b"))
+    assert_frame_equal(expected_ab, gdf.dissolve(level=[0, 1]))
+    assert_frame_equal(expected_ab, gdf.dissolve(level=["a", "b"]))
+
+
+def test_dissolve_sort():
+    gdf = geopandas.GeoDataFrame(
+        {
+            "a": [2, 1, 1],
+            "geometry": geopandas.array.from_wkt(
+                ["POINT (0 0)", "POINT (1 1)", "POINT (2 2)"]
+            ),
+        }
+    )
+
+    expected_unsorted = geopandas.GeoDataFrame(
+        {
+            "a": [2, 1],
+            "geometry": geopandas.array.from_wkt(
+                ["POINT (0 0)", "MULTIPOINT (1 1, 2 2)"]
+            ),
+        }
+    ).set_index("a")
+    expected_sorted = expected_unsorted.sort_index()
+
+    assert_frame_equal(expected_sorted, gdf.dissolve("a"))
+    assert_frame_equal(expected_unsorted, gdf.dissolve("a", sort=False))
+
+
+@pytest.mark.skipif(
+    not compat.PANDAS_GE_025,
+    reason="'observed' param behavior changed in pandas 0.25.0",
+)
+def test_dissolve_categorical():
+    gdf = geopandas.GeoDataFrame(
+        {
+            "cat": pd.Categorical(["a", "a", "b", "b"]),
+            "noncat": [1, 1, 1, 2],
+            "to_agg": [1, 2, 3, 4],
+            "geometry": geopandas.array.from_wkt(
+                ["POINT (0 0)", "POINT (1 1)", "POINT (2 2)", "POINT (3 3)"]
+            ),
+        }
+    )
+
+    # when observed=False we get an additional observation
+    # that wasn't in the original data
+    expected_gdf_observed_false = geopandas.GeoDataFrame(
+        {
+            "cat": pd.Categorical(["a", "a", "b", "b"]),
+            "noncat": [1, 2, 1, 2],
+            "geometry": geopandas.array.from_wkt(
+                [
+                    "MULTIPOINT (0 0, 1 1)",
+                    None,
+                    "POINT (2 2)",
+                    "POINT (3 3)",
+                ]
+            ),
+            "to_agg": [1, None, 3, 4],
+        }
+    ).set_index(["cat", "noncat"])
+
+    # when observed=True we do not get any additional observations
+    expected_gdf_observed_true = geopandas.GeoDataFrame(
+        {
+            "cat": pd.Categorical(["a", "b", "b"]),
+            "noncat": [1, 1, 2],
+            "geometry": geopandas.array.from_wkt(
+                ["MULTIPOINT (0 0, 1 1)", "POINT (2 2)", "POINT (3 3)"]
+            ),
+            "to_agg": [1, 3, 4],
+        }
+    ).set_index(["cat", "noncat"])
+
+    assert_frame_equal(expected_gdf_observed_false, gdf.dissolve(["cat", "noncat"]))
+    assert_frame_equal(
+        expected_gdf_observed_true, gdf.dissolve(["cat", "noncat"], observed=True)
+    )
+
+
+@pytest.mark.skipif(
+    not compat.PANDAS_GE_11, reason="dropna groupby kwarg added in pandas 1.1.0"
+)
+def test_dissolve_dropna():
+    gdf = geopandas.GeoDataFrame(
+        {
+            "a": [1, 1, None],
+            "geometry": geopandas.array.from_wkt(
+                ["POINT (0 0)", "POINT (1 1)", "POINT (2 2)"]
+            ),
+        }
+    )
+
+    expected_with_na = geopandas.GeoDataFrame(
+        {
+            "a": [1.0, np.nan],
+            "geometry": geopandas.array.from_wkt(
+                ["MULTIPOINT (0 0, 1 1)", "POINT (2 2)"]
+            ),
+        }
+    ).set_index("a")
+    expected_no_na = geopandas.GeoDataFrame(
+        {
+            "a": [1.0],
+            "geometry": geopandas.array.from_wkt(["MULTIPOINT (0 0, 1 1)"]),
+        }
+    ).set_index("a")
+
+    assert_frame_equal(expected_with_na, gdf.dissolve("a", dropna=False))
+    assert_frame_equal(expected_no_na, gdf.dissolve("a"))
+
+
+@pytest.mark.skipif(
+    compat.PANDAS_GE_11, reason="dropna warning is only emitted if pandas < 1.1.0"
+)
+def test_dissolve_dropna_warn(nybb_polydf):
+    # No warning with default params
+    with pytest.warns(None) as record:
+        nybb_polydf.dissolve()
+
+    for r in record:
+        assert "dropna kwarg is not supported" not in str(r.message)
+
+    # Warning is emitted with non-default dropna value
+    with pytest.warns(
+        UserWarning, match="dropna kwarg is not supported for pandas < 1.1.0"
+    ):
+        nybb_polydf.dissolve(dropna=False)