Repository: geopandas/geopandas
Pull request #3504 (2025-02-04 20:25:40)

ENH: improve performance of overlay/identity
Remark:
- The speedup has as side-effect that e.g. int64 columns with NAN values in some situations will retain the original int64 dtype instead of switching to float64

Solves #3295


Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4224727e..fe41c2be 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,6 +11,7 @@ New features and improvements:
 - Add ``grid_size`` parameter to ``union_all`` and ``dissolve`` (#3445).
 - `GeoDataFrame.plot` now supports `pd.Index` as an input for the `column` keyword (#3463).
 - Avoid change of the plot aspect when plotting missing values (#3438).
+- Improve performance of `overlay` with `how=identity` (#3504).
 
 Bug fixes:
 
diff --git a/geopandas/geodataframe.py b/geopandas/geodataframe.py
index f716ea8c..dfe68ba7 100644
--- a/geopandas/geodataframe.py
+++ b/geopandas/geodataframe.py
@@ -2831,11 +2831,11 @@ chicago_w_groceries[chicago_w_groceries["community"] == "UPTOWN"]
 
         >>> df1.overlay(df2, how='identity')
            df1_data  df2_data                                           geometry
-        0       1.0       1.0                POLYGON ((2 2, 2 1, 1 1, 1 2, 2 2))
-        1       2.0       1.0                POLYGON ((2 2, 2 3, 3 3, 3 2, 2 2))
-        2       2.0       2.0                POLYGON ((4 4, 4 3, 3 3, 3 4, 4 4))
-        3       1.0       NaN      POLYGON ((2 0, 0 0, 0 2, 1 2, 1 1, 2 1, 2 0))
-        4       2.0       NaN  MULTIPOLYGON (((3 4, 3 3, 2 3, 2 4, 3 4)), ((4...
+        0         1       1.0                POLYGON ((2 2, 2 1, 1 1, 1 2, 2 2))
+        1         2       1.0                POLYGON ((2 2, 2 3, 3 3, 3 2, 2 2))
+        2         2       2.0                POLYGON ((4 4, 4 3, 3 3, 3 4, 4 4))
+        3         1       NaN      POLYGON ((2 0, 0 0, 0 2, 1 2, 1 1, 2 1, 2 0))
+        4         2       NaN  MULTIPOLYGON (((3 4, 3 3, 2 3, 2 4, 3 4)), ((4...
 
         See also
         --------
diff --git a/geopandas/tools/overlay.py b/geopandas/tools/overlay.py
index 6332d425..66338419 100644
--- a/geopandas/tools/overlay.py
+++ b/geopandas/tools/overlay.py
@@ -110,6 +110,21 @@ def _overlay_difference(df1, df2):
     return dfdiff
 
 
+def _overlay_identity(df1, df2):
+    """
+    Overlay Identity operation used in overlay function.
+    """
+    dfintersection = _overlay_intersection(df1, df2)
+    dfdifference = _overlay_difference(df1, df2)
+    dfdifference = _ensure_geometry_column(dfdifference)
+    result = pd.concat([dfintersection, dfdifference], ignore_index=True, sort=False)
+    # keep geometry column last
+    columns = list(dfintersection.columns)
+    columns.remove("geometry")
+    columns.append("geometry")
+    return result.reindex(columns=columns)
+
+
 def _overlay_symmetric_diff(df1, df2):
     """
     Overlay Symmetric Difference operation used in overlay function
@@ -225,11 +240,11 @@ def overlay(df1, df2, how="intersection", keep_geom_type=None, make_valid=True):
 
     >>> geopandas.overlay(df1, df2, how='identity')
        df1_data  df2_data                                           geometry
-    0       1.0       1.0                POLYGON ((2 2, 2 1, 1 1, 1 2, 2 2))
-    1       2.0       1.0                POLYGON ((2 2, 2 3, 3 3, 3 2, 2 2))
-    2       2.0       2.0                POLYGON ((4 4, 4 3, 3 3, 3 4, 4 4))
-    3       1.0       NaN      POLYGON ((2 0, 0 0, 0 2, 1 2, 1 1, 2 1, 2 0))
-    4       2.0       NaN  MULTIPOLYGON (((3 4, 3 3, 2 3, 2 4, 3 4)), ((4...
+    0         1       1.0                POLYGON ((2 2, 2 1, 1 1, 1 2, 2 2))
+    1         2       1.0                POLYGON ((2 2, 2 3, 3 3, 3 2, 2 2))
+    2         2       2.0                POLYGON ((4 4, 4 3, 3 3, 3 4, 4 4))
+    3         1       NaN      POLYGON ((2 0, 0 0, 0 2, 1 2, 1 1, 2 1, 2 0))
+    4         2       NaN  MULTIPOLYGON (((3 4, 3 3, 2 3, 2 4, 3 4)), ((4...
 
     See also
     --------
@@ -333,8 +348,7 @@ def overlay(df1, df2, how="intersection", keep_geom_type=None, make_valid=True):
         elif how == "union":
             result = _overlay_union(df1, df2)
         elif how == "identity":
-            dfunion = _overlay_union(df1, df2)
-            result = dfunion[dfunion["__idx1"].notnull()].copy()
+            result = _overlay_identity(df1, df2)
 
         if how in ["intersection", "symmetric_difference", "union", "identity"]:
             result.drop(["__idx1", "__idx2"], axis=1, inplace=True)


Test patch:
diff --git a/geopandas/tests/test_overlay.py b/geopandas/tests/test_overlay.py
index c9f8dad0..ee35bcc4 100644
--- a/geopandas/tests/test_overlay.py
+++ b/geopandas/tests/test_overlay.py
@@ -90,7 +90,6 @@ def test_overlay(dfs_index, how):
         expected = pd.concat(
             [expected_intersection, expected_difference], ignore_index=True, sort=False
         )
-        expected["col1"] = expected["col1"].astype(float)
     else:
         expected = _read(how)
 
@@ -174,6 +173,10 @@ def test_overlay_nybb(how, nybb_filename):
 
     if how == "identity":
         expected = expected[expected.BoroCode.notnull()].copy()
+        boro_code_dtype = result["BoroCode"].dtype
+        if boro_code_dtype in ("int32", "int64"):
+            # Depending on the pandas version, the dtype might be int32 or int64
+            expected["BoroCode"] = expected["BoroCode"].astype(boro_code_dtype)
 
     # Order GeoDataFrames
     expected = expected.sort_values(cols).reset_index(drop=True)
@@ -786,7 +789,7 @@ def test_non_overlapping(how):
     elif how == "identity":
         expected = GeoDataFrame(
             {
-                "col1": [1.0],
+                "col1": [1],
                 "col2": [np.nan],
                 "geometry": [p1],
             }