Repository: geopandas/geopandas

Issue:
TST: failures with shapely and GEOS 3.9
CI started coming with GEOS 3.9 today which caused a lot of errors if we use shapely backend. Most of them are related to overlay, so I guess it is caused by the new `OverlayNG`, which returns slightly different geometries.

https://github.com/geopandas/geopandas/runs/1854075806


Discussion:
The overlay failures are caused by a different order of coordinates, everything else looks the same to me. Since we use `geom_almost_equals` in the check, we need equality of order, so it is a matter of fixing the tests.

And the issue with `polygonpatch` has the same origin. We use `difference` to get the geometry, and new GEOS gives us 195 vertices instead of 198. Since all are circles, I don't think it matters.
Hmm, it seems that we'll need a test which checks that geometry is purely almost equal in a way where order of vertices and precision of coordinates does not matter (now we can do one or the other).
> Hmm, it seems that we'll need a test which checks that geometry is purely almost equal in a way where order of vertices and precision of coordinates does not matter (now we can do one or the other).

In principle, the easiest way to do this is to "normalize" the geometries before comparing. Problem is that this is only available for pygeos and not shapely ..
It should actually be relatively easy to use shapely's geos internals to expose normalize, will do a PR tomorrow.

Resolving patch:
diff --git a/geopandas/_compat.py b/geopandas/_compat.py
--- a/geopandas/_compat.py
+++ b/geopandas/_compat.py
@@ -6,6 +6,8 @@
 
 import pandas as pd
 import shapely
+import shapely.geos
+
 
 # -----------------------------------------------------------------------------
 # pandas compat
@@ -26,6 +28,8 @@
 SHAPELY_GE_18 = str(shapely.__version__) >= LooseVersion("1.8")
 SHAPELY_GE_20 = str(shapely.__version__) >= LooseVersion("2.0")
 
+GEOS_GE_390 = shapely.geos.geos_version >= (3, 9, 0)
+
 
 HAS_PYGEOS = None
 USE_PYGEOS = None
diff --git a/geopandas/_vectorized.py b/geopandas/_vectorized.py
--- a/geopandas/_vectorized.py
+++ b/geopandas/_vectorized.py
@@ -828,6 +828,34 @@ def simplify(data, tolerance, preserve_topology=True):
         return out
 
 
+def _shapely_normalize(geom):
+    """
+    Small helper function for now because it is not yet available in Shapely.
+    """
+    from shapely.geos import lgeos
+    from shapely.geometry.base import geom_factory
+    from ctypes import c_void_p, c_int
+
+    lgeos._lgeos.GEOSNormalize_r.restype = c_int
+    lgeos._lgeos.GEOSNormalize_r.argtypes = [c_void_p, c_void_p]
+
+    geom_cloned = lgeos.GEOSGeom_clone(geom._geom)
+    lgeos._lgeos.GEOSNormalize_r(lgeos.geos_handle, geom_cloned)
+    return geom_factory(geom_cloned)
+
+
+def normalize(data):
+    if compat.USE_PYGEOS:
+        return pygeos.normalize(data)
+    else:
+        out = np.empty(len(data), dtype=object)
+        with compat.ignore_shapely2_warnings():
+            out[:] = [
+                _shapely_normalize(geom) if geom is not None else None for geom in data
+            ]
+        return out
+
+
 def project(data, other, normalized=False):
     if compat.USE_PYGEOS:
         return pygeos.line_locate_point(data, other, normalize=normalized)


Test patch:
diff --git a/geopandas/testing.py b/geopandas/testing.py
--- a/geopandas/testing.py
+++ b/geopandas/testing.py
@@ -7,6 +7,7 @@
 
 from geopandas import GeoDataFrame, GeoSeries
 from geopandas.array import GeometryDtype
+from geopandas import _vectorized
 
 
 def _isna(this):
@@ -67,6 +68,7 @@ def assert_geoseries_equal(
     check_less_precise=False,
     check_geom_type=False,
     check_crs=True,
+    normalize=False,
 ):
     """
     Test util for checking that two GeoSeries are equal.
@@ -89,6 +91,10 @@ def assert_geoseries_equal(
     check_crs: bool, default True
         If `check_series_type` is True, then also check that the
         crs matches.
+    normalize: bool, default False
+        If True, normalize the geometries before comparing equality.
+        Typically useful with ``check_less_precise=True``, which uses
+        ``geom_almost_equals`` and requires exact coordinate order.
     """
     assert len(left) == len(right), "%d != %d" % (len(left), len(right))
 
@@ -120,6 +126,10 @@ def assert_geoseries_equal(
             right.type,
         )
 
+    if normalize:
+        left = GeoSeries(_vectorized.normalize(left.array.data))
+        right = GeoSeries(_vectorized.normalize(right.array.data))
+
     if not check_crs:
         with warnings.catch_warnings():
             warnings.filterwarnings("ignore", "CRS mismatch", UserWarning)
@@ -145,6 +155,7 @@ def assert_geodataframe_equal(
     check_less_precise=False,
     check_geom_type=False,
     check_crs=True,
+    normalize=False,
 ):
     """
     Check that two GeoDataFrames are equal/
@@ -168,6 +179,10 @@ def assert_geodataframe_equal(
     check_crs: bool, default True
         If `check_frame_type` is True, then also check that the
         crs matches.
+    normalize: bool, default False
+        If True, normalize the geometries before comparing equality.
+        Typically useful with ``check_less_precise=True``, which uses
+        ``geom_almost_equals`` and requires exact coordinate order.
     """
     try:
         # added from pandas 0.20
@@ -217,6 +232,7 @@ def assert_geodataframe_equal(
             assert_geoseries_equal(
                 left[col],
                 right[col],
+                normalize=normalize,
                 check_dtype=check_dtype,
                 check_less_precise=check_less_precise,
                 check_geom_type=check_geom_type,
diff --git a/geopandas/tests/test_overlay.py b/geopandas/tests/test_overlay.py
--- a/geopandas/tests/test_overlay.py
+++ b/geopandas/tests/test_overlay.py
@@ -107,19 +107,23 @@ def _read(name):
 def test_overlay_nybb(how):
     polydf = read_file(geopandas.datasets.get_path("nybb"))
 
-    # construct circles dataframe
-    N = 10
-    b = [int(x) for x in polydf.total_bounds]
-    polydf2 = GeoDataFrame(
-        [
-            {"geometry": Point(x, y).buffer(10000), "value1": x + y, "value2": x - y}
-            for x, y in zip(
-                range(b[0], b[2], int((b[2] - b[0]) / N)),
-                range(b[1], b[3], int((b[3] - b[1]) / N)),
-            )
-        ],
-        crs=polydf.crs,
-    )
+    # The circles have been constructed and saved at the time the expected
+    # results were created (exact output of buffer algorithm can slightly
+    # change over time -> use saved ones)
+    # # construct circles dataframe
+    # N = 10
+    # b = [int(x) for x in polydf.total_bounds]
+    # polydf2 = GeoDataFrame(
+    #     [
+    #         {"geometry": Point(x, y).buffer(10000), "value1": x + y, "value2": x - y}
+    #         for x, y in zip(
+    #             range(b[0], b[2], int((b[2] - b[0]) / N)),
+    #             range(b[1], b[3], int((b[3] - b[1]) / N)),
+    #         )
+    #     ],
+    #     crs=polydf.crs,
+    # )
+    polydf2 = read_file(os.path.join(DATA, "nybb_qgis", "polydf2.shp"))
 
     result = overlay(polydf, polydf2, how=how)
 
@@ -186,12 +190,23 @@ def test_overlay_nybb(how):
         result.geometry.bounds, expected.geometry.bounds, check_less_precise=True
     )
 
-    # now drop multipolygons
-    result.geometry[result.geometry.geom_type == "MultiPolygon"] = None
-    expected.geometry[expected.geometry.geom_type == "MultiPolygon"] = None
+    # There are two cases where the multipolygon have a different number
+    # of sub-geometries -> not solved by normalize (and thus drop for now)
+    if how == "symmetric_difference":
+        expected.loc[9, "geometry"] = None
+        result.loc[9, "geometry"] = None
+
+    if how == "union":
+        expected.loc[24, "geometry"] = None
+        result.loc[24, "geometry"] = None
 
     assert_geodataframe_equal(
-        result, expected, check_crs=False, check_column_type=False
+        result,
+        expected,
+        normalize=True,
+        check_crs=False,
+        check_column_type=False,
+        check_less_precise=True,
     )
 
 
@@ -244,7 +259,11 @@ def test_overlay_overlap(how):
         result = result.sort_values(["col1", "col2"]).reset_index(drop=True)
 
     assert_geodataframe_equal(
-        result, expected, check_column_type=False, check_less_precise=True
+        result,
+        expected,
+        normalize=True,
+        check_column_type=False,
+        check_less_precise=True,
     )
 
 
@@ -469,6 +488,7 @@ def test_overlay_strict(how, keep_geom_type, geom_types):
         assert_geodataframe_equal(
             result,
             expected,
+            normalize=True,
             check_column_type=False,
             check_less_precise=True,
             check_crs=False,
diff --git a/geopandas/tests/test_plotting.py b/geopandas/tests/test_plotting.py
--- a/geopandas/tests/test_plotting.py
+++ b/geopandas/tests/test_plotting.py
@@ -1490,7 +1490,10 @@ def test_polygon_patch():
     patch = _PolygonPatch(polygon)
     assert isinstance(patch, PathPatch)
     path = patch.get_path()
-    assert len(path.vertices) == len(path.codes) == 198
+    if compat.GEOS_GE_390:
+        assert len(path.vertices) == len(path.codes) == 195
+    else:
+        assert len(path.vertices) == len(path.codes) == 198
 
 
 def _check_colors(N, actual_colors, expected_colors, alpha=None):
diff --git a/geopandas/tests/test_sindex.py b/geopandas/tests/test_sindex.py
--- a/geopandas/tests/test_sindex.py
+++ b/geopandas/tests/test_sindex.py
@@ -412,10 +412,14 @@ def test_query_sorting(self, sort, expected):
 
         test_geo = test_df.geometry.values.data[0]
         res = tree_df.sindex.query(test_geo, sort=sort)
+
+        # asserting the same elements
+        assert sorted(res) == sorted(expected)
+        # asserting the exact array can fail if sort=False
         try:
             assert_array_equal(res, expected)
         except AssertionError as e:
-            if not compat.USE_PYGEOS and sort is False:
+            if sort is False:
                 pytest.xfail(
                     "rtree results are known to be unordered, see "
                     "https://github.com/geopandas/geopandas/issues/1337\n"
@@ -649,10 +653,15 @@ def test_query_bulk_sorting(self, sort, expected):
         test_df = geopandas.GeoDataFrame(geometry=test_polys)
 
         res = tree_df.sindex.query_bulk(test_df.geometry, sort=sort)
+
+        # asserting the same elements
+        assert sorted(res[0]) == sorted(expected[0])
+        assert sorted(res[1]) == sorted(expected[1])
+        # asserting the exact array can fail if sort=False
         try:
             assert_array_equal(res, expected)
         except AssertionError as e:
-            if not compat.USE_PYGEOS and sort is False:
+            if sort is False:
                 pytest.xfail(
                     "rtree results are known to be unordered, see "
                     "https://github.com/geopandas/geopandas/issues/1337\n"