Repository: geopandas/geopandas

Issue:
BUG: `convert_dtypes()` converts GeoDataFrame to DataFrame
- [X] I have checked that this issue has not already been reported.

- [X] I have confirmed this bug exists on the latest version of geopandas.

- [ ] (optional) I have confirmed this bug exists on the master branch of geopandas.

---

#### Code Sample, a copy-pastable example

```python
import pandas as pd
import fiona # AttributeError if not importing fiona before gpd
import geopandas as gpd
from geopandas.testing import assert_geodataframe_equal

def df():
    return pd.DataFrame(
        {
            "City": ["Buenos Aires", "Brasilia", "Santiago", "Bogota", "Caracas"],
            "Country": ["Argentina", "Brazil", "Chile", "Colombia", "Venezuela"],
            "Latitude": [-34.58, -15.78, -33.45, 4.60, 10.48],
            "Longitude": [-58.66, -47.91, -70.66, -74.08, -66.86],
        }
    )

def gdf_points_from_xy(df):
    return gpd.GeoDataFrame(df, geometry=gpd.points_from_xy(df.Longitude, df.Latitude))

def test_convert_dtypes_before_gdf():
    result = df().convert_dtypes().pipe(gdf_points_from_xy)
    assert isinstance(result, gpd.GeoDataFrame)
    # --> no error

def test_convert_dtypes_after_gdf():
    result = df().pipe(gdf_points_from_xy).convert_dtypes()
    assert isinstance(result, gpd.GeoDataFrame)
    # --> AssertionError

def test_convert_dtypes_expectation():
    expected = df().convert_dtypes().pipe(gdf_points_from_xy)

    result = df().pipe(gdf_points_from_xy).convert_dtypes()

    assert_geodataframe_equal(result, expected)
    #--> AssertionError
```

#### Problem description

Calling `.convert_dtypes()` on a GeoDataFrame turns it into a regular DataFrame.

```python
AssertionError: assert isinstance(result, GeoDataFrame)
```

#### Expected Output
I expect `convert_dtypes` to not change frame type.

The problem may be with the [original method](https://github.com/pandas-dev/pandas/blob/c5d0d138cf1162cea985741cbbbaea3eef6ca8c3/pandas/core/generic.py#L6138). I suspect it is because of `return concat(results, axis=1, copy=False)`. As the pandas repo does not know about other frame types, I suspect the fix should lie in this repo.

#### Output of ``geopandas.show_versions()``

<details>

SYSTEM INFO
-----------
python     : 3.8.4 (default, Jan 11 2021, 16:58:12)  [Clang 12.0.0 (clang-1200.0.32.28)]
executable : /Users/adriantofting/Library/Caches/pypoetry/virtualenvs/gpd-test-qc5veAeh-py3.8/bin/python
machine    : macOS-11.1-x86_64-i386-64bit

GEOS, GDAL, PROJ INFO
---------------------
GEOS       : 3.9.1
GEOS lib   : /usr/local/Cellar/geos/3.9.1/lib/libgeos_c.dylib
GDAL       : 3.2.1
GDAL data dir: None
PROJ       : 7.2.1
PROJ data dir: /usr/local/share/proj

PYTHON DEPENDENCIES
-------------------
geopandas  : 0.9.0
pandas     : 1.2.3
fiona      : 1.8.18
numpy      : 1.20.1
shapely    : 1.7.1
rtree      : None
pyproj     : 3.0.0.post1
matplotlib : None
mapclassify: None
geopy      : None
psycopg2   : None
geoalchemy2: None
pyarrow    : None
pygeos     : None

</details>

#### Environment (poetry)

<details>

```toml
#pyproject.toml
[tool.poetry]
name = "gpd_test"
version = "0.1.0"
description = ""

[tool.poetry.dependencies]
python = "^3.8"
geopandas = "0.9.0"

[tool.poetry.dev-dependencies]
jupyterlab = "^3.0.9"

[build-system]
requires = ["poetry-core>=1.0.0"]
build-backend = "poetry.core.masonry.api"
```

</details



Discussion:
Thanks for the report! This is indeed something we need to fix in GeoPandas as pandas internally split the DataFrame into a list of (Geo)Series and concatenate them together, which essentially creates a brand new object.

We need to override the method and ensure that we preserve GeoDataFrame as well as our metadata (active geometry, CRS), which are currently also lost.
convert_dtypes is declared as final, at least in the last versions of pandas, so it should not be overridden (I don't know if python actually prevents it or it is just a recommendation). In any case, I think this does the trick:
```python
def convert_dtypes_geopandas(gdf):
    return pd.concat((gdf[['geometry']],
                      gdf.drop(columns=['geometry']).convert_dtypes()),
                      axis=1)
```
Just a note on reproducing this (because I couldn't in my mock up at first), order matters to the result of convert dtypes - because order matters in concat:
```python
In [8]: type(df().pipe(gdf_points_from_xy)[['geometry', 'Country']].convert_dtypes())
Out[8]: geopandas.geodataframe.GeoDataFrame

In [9]: type(df().pipe(gdf_points_from_xy)[['Country', 'geometry']].convert_dtypes())
Out[9]: pandas.core.frame.DataFrame
```

Interesting find! Indeed, when ordering the columns by putting 'geometry' first, the assertion holds. 

But when checking that the frames are equal, I get the error `KeyError: '[None] not found in axis'`. Don't see any difference in them, though.

```python
def test_convert_dtypes_after_gdf():
    result = df().pipe(gdf_points_from_xy)[["geometry", "City", "Country"]].convert_dtypes()
    assert isinstance(result, gpd.GeoDataFrame)
    # --> no error


def test_convert_dtypes_expectation():
    ordered_columns = ["geometry", "City", "Country"]

    expected = (
        df()
        .convert_dtypes()
        .pipe(gdf_points_from_xy)[ordered_columns]
    )

    result = (
        df()
        .pipe(gdf_points_from_xy)[ordered_columns]
        .convert_dtypes()
    )

    assert_geodataframe_equal(result, expected)
    # --> AssertionError: KeyError: '[None] not found in axis'
```

<details>
<summary>Full stacktrace</summary>

```python
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
packages/geopandas/testing.py:283: in assert_geodataframe_equal
    left2 = left.drop([left._geometry_column_name], axis=1)
packages/pandas/util/_decorators.py:311: in wrapper
    return func(*args, **kwargs)
packages/pandas/core/frame.py:4901: in drop
    return super().drop(
packages/pandas/core/generic.py:4150: in drop
    obj = obj._drop_axis(labels, axis, level=level, errors=errors)
packages/pandas/core/generic.py:4185: in _drop_axis
    new_axis = axis.drop(labels, errors=errors)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = Index(['geometry', 'City', 'Country'], dtype='object')
labels = array([None], dtype=object), errors = 'raise'

    def drop(self, labels, errors: str_t = "raise") -> Index:
        """
        Make new Index with passed list of labels deleted.
    
        Parameters
        ----------
        labels : array-like
        errors : {'ignore', 'raise'}, default 'raise'
            If 'ignore', suppress error and existing labels are dropped.
    
        Returns
        -------
        dropped : Index
            Will be same type as self, except for RangeIndex.
    
        Raises
        ------
        KeyError
            If not all of the labels are found in the selected axis
        """
        arr_dtype = "object" if self.dtype == "object" else None
        labels = com.index_labels_to_array(labels, dtype=arr_dtype)
        indexer = self.get_indexer_for(labels)
        mask = indexer == -1
        if mask.any():
            if errors != "ignore":
>               raise KeyError(f"{labels[mask]} not found in axis")
E               KeyError: '[None] not found in axis'

packages/pandas/core/indexes/base.py:6018: KeyError

```

</details>
I think this can actually also be considered a bug in pandas itself, as it should use `_constructor` to recreate the resulting dataframe. Of course, on the short-term, we can also fix it in geopandas by overriding the method. PR welcome for that!

@damanad regarding pandas denoting the method as `@final`, pandas uses that to indicate that the method doesn't get overriden internally in pandas itself. We have several methods in geopandas that we override from pandas (that already might have this decorator, didn't check). So in general I think we can ignore this. If it would give problems with typing (validating type annotations in geopandas), we might need to ask pandas to not use it like that (but no typing expert).

> But when checking that the frames are equal, I get the error `KeyError: '[None] not found in axis'`. Don't see any difference in them, though.

Although, when putting the geometry column first, it now returns a GeoDataFrame, it's apparently still an "invalid" GeoDataFrame, in the sense that the `_geometry_column_name` has not been set properly. I think this can be seen as a separate bug in `pd.concat([geoseries, series, ..], axis=1)`
@jorisvandenbossche it looks like pandas is using the _constructor, but before that they cast it to DataFrame ([source code](https://github.com/pandas-dev/pandas/blob/f482bb751f6e230bc74fefa385a68bc4998d6842/pandas/core/reshape/concat.py#L513))
. What do you think is the reason for that?
convert_dtypes [actually calls concat internally](https://github.com/pandas-dev/pandas/blob/95d2667dd88ba71593bcaa7dd2ad51f7e290325d/pandas/core/generic.py#L6264), so maybe concat is the root to problem.
That `cast` doesn't actually "cast" anything in runtime, but only does that to help the type checker.

But yes, it's the facto that it _only_ calls concat that is the problem here. Ideally pandas should still pass the result of concat to `_constructor` or call `__finalize__` on it (this second option might actually be sufficient).
Thanks for the clarification. Is this issue indeed what we need?
https://github.com/pandas-dev/pandas/issues/28283
Seems like this is [fixed for append](https://github.com/pandas-dev/pandas/blob/1b6cb0f45c36b79ef4df27b86f3785684e147504/pandas/core/frame.py#L8947), so is the fix as simple as this?
```python
8947    return concat(results, axis=1, copy=False).__finalize__(self, method="concat")
```
I'm pretty sure finalize on it's own won't be enough, because the return type of concat depends on the return type of the first argument in the list of frames - which is why order matters in what I posted above. So if concat returns a DataFrame,
`DataFrame.__finalize` will be called  (as opposed to `GeoDataFrame.__finalize__` and we won't have any mechanism to turn that into a `GeoDataFrame`.

Resolving patch:
diff --git a/geopandas/geodataframe.py b/geopandas/geodataframe.py
--- a/geopandas/geodataframe.py
+++ b/geopandas/geodataframe.py
@@ -1652,6 +1652,34 @@ def astype(self, dtype, copy=True, errors="raise", **kwargs):
         # do not return a GeoDataFrame
         return pd.DataFrame(df)
 
+    def convert_dtypes(self, *args, **kwargs):
+        """
+        Convert columns to best possible dtypes using dtypes supporting ``pd.NA``.
+
+        Always returns a GeoDataFrame as no conversions are applied to the
+        geometry column.
+
+        See the pandas.DataFrame.convert_dtypes docstring for more details.
+
+        Returns
+        -------
+        GeoDataFrame
+
+        """
+        # Overridden to fix GH1870, that return type is not preserved always
+        # (and where it was, geometry col was not)
+
+        if not compat.PANDAS_GE_10:
+            raise NotImplementedError(
+                "GeoDataFrame.convert_dtypes requires pandas >= 1.0"
+            )
+
+        return GeoDataFrame(
+            super().convert_dtypes(*args, **kwargs),
+            geometry=self.geometry.name,
+            crs=self.crs,
+        )
+
     def to_postgis(
         self,
         name,


Test patch:
diff --git a/geopandas/tests/test_pandas_methods.py b/geopandas/tests/test_pandas_methods.py
--- a/geopandas/tests/test_pandas_methods.py
+++ b/geopandas/tests/test_pandas_methods.py
@@ -262,6 +262,42 @@ def test_astype_invalid_geodataframe():
     assert res["a"].dtype == object
 
 
+@pytest.mark.xfail(
+    not compat.PANDAS_GE_10,
+    reason="Convert dtypes new in pandas 1.0",
+    raises=NotImplementedError,
+)
+def test_convert_dtypes(df):
+    # https://github.com/geopandas/geopandas/issues/1870
+
+    # Test geometry col is first col, first, geom_col_name=geometry
+    # (order is important in concat, used internally)
+    res1 = df.convert_dtypes()  # note res1 done first for pandas < 1 xfail check
+
+    expected1 = GeoDataFrame(
+        pd.DataFrame(df).convert_dtypes(), crs=df.crs, geometry=df.geometry.name
+    )
+
+    # Checking type and metadata are right
+    assert_geodataframe_equal(expected1, res1)
+
+    # Test geom last, geom_col_name=geometry
+    res2 = df[["value1", "value2", "geometry"]].convert_dtypes()
+    assert_geodataframe_equal(expected1[["value1", "value2", "geometry"]], res2)
+
+    # Test again with crs set and custom geom col name
+    df2 = df.set_crs(epsg=4326).rename_geometry("points")
+    expected2 = GeoDataFrame(
+        pd.DataFrame(df2).convert_dtypes(), crs=df2.crs, geometry=df2.geometry.name
+    )
+    res3 = df2.convert_dtypes()
+    assert_geodataframe_equal(expected2, res3)
+
+    # Test geom last, geom_col=geometry
+    res4 = df2[["value1", "value2", "points"]].convert_dtypes()
+    assert_geodataframe_equal(expected2[["value1", "value2", "points"]], res4)
+
+
 def test_to_csv(df):
 
     exp = (