Repository: PyPSA/PyPSA

Issue:
Performance: Faster network.copy(), perhaps using copy.deepcopy
At present, copying a network with `pypsa.Network().copy()` takes a surprisingly long time. For some workflows this actually seems to be a performance bottleneck.

I noticed that using the builtin `copy.deepcopy` seems to do the same job much faster:

```python
import pypsa
import copy

n = pypsa.examples.ac_dc_meshed()
n2 = n.copy()  # takes about 158ms
n3 = copy.deepcopy(n)  # takes about 5 ms
```

Would this be a suitable alternative implementation? Are there any issues with using `copy.deepcopy`?

I can look into making a PR if that would be of interest.


Discussion:

Patch:
diff --git a/doc/release_notes.rst b/doc/release_notes.rst
index 2196ccf1b..f5074df3a 100644
--- a/doc/release_notes.rst
+++ b/doc/release_notes.rst
@@ -12,15 +12,21 @@ Upcoming Release
 
 * HiGHS becomes the new default solver for ``n.optimize()``.
 
+* Add functionality to compare two networks for equality via equality operator (``==``). 
+  (https://github.com/PyPSA/PyPSA/pull/924)
+
 * Bugfix: Retain investment periods and weightings when clustering networks.
 
-* When adding components with bus ports greater than 1, e.g. `bus2`, pypsa checks if the bus exists and prints a warning if it does not.
+* When adding components with bus ports greater than 1, e.g. `bus2`, pypsa checks if 
+  the bus exists and prints a warning if it does not.
 
-* When adding bus ports on the fly with `add` methods, the dtype of the freshly created column is now fixed to `string`.
+* When adding bus ports on the fly with `add` methods, the dtype of the freshly created 
+  column is now fixed to `string`.
 
 * Add functionality to provide individual colors for lines in legend during plot.
 
-* Also check for missing values of default attributes in the `n.consistency_check()` function. (https://github.com/PyPSA/PyPSA/pull/903)
+* Also check for missing values of default attributes in the `n.consistency_check()` 
+  function. (https://github.com/PyPSA/PyPSA/pull/903)
 
 PyPSA 0.28.0 (8th May 2024)
 =================================
diff --git a/pypsa/components.py b/pypsa/components.py
index 4dfa58e4c..cfe628a27 100644
--- a/pypsa/components.py
+++ b/pypsa/components.py
@@ -1,8 +1,11 @@
 """Power system components."""
 
+import copy
 import logging
 import os
+import warnings
 from collections import namedtuple
+from collections.abc import Collection
 from pathlib import Path
 from typing import Union
 from weakref import ref
@@ -74,6 +77,7 @@
 from pypsa.statistics import StatisticsAccessor
 
 logger = logging.getLogger(__name__)
+warnings.simplefilter("always", DeprecationWarning)
 
 
 dir_name = os.path.dirname(__file__)
@@ -375,6 +379,54 @@ def __add__(self, other):
         """Merge all components of two networks."""
         self.merge(other)
 
+    def __eq__(self, other):
+        """Check for equality of two networks."""
+
+        def equals(a, b):
+            assert isinstance(a, type(b)), f"Type mismatch: {type(a)} != {type(b)}"
+            # Classes with equality methods
+            if isinstance(a, np.ndarray):
+                if not np.array_equal(a, b):
+                    return False
+            elif isinstance(a, (pd.DataFrame, pd.Series, pd.Index)):
+                if not a.equals(b):
+                    return False
+            # Iterators
+            elif isinstance(a, (dict, Dict)):
+                for k, v in a.items():
+                    if not equals(v, b[k]):
+                        return False
+            elif isinstance(a, (list, tuple)):
+                for i, v in enumerate(a):
+                    if not equals(v, b[i]):
+                        return False
+            # Ignore for now
+            elif isinstance(
+                value, (OptimizationAccessor, ClusteringAccessor, StatisticsAccessor)
+            ):
+                pass
+            # Nans
+            elif pd.isna(a) and pd.isna(b):
+                pass
+            else:
+                if a != b:
+                    return False
+
+            return True
+
+        if isinstance(other, self.__class__):
+            for key, value in self.__dict__.items():
+                if not equals(value, other.__dict__[key]):
+                    return False
+        else:
+            logger.warning(
+                "Can only compare two pypsa.Network objects with each other. Got %s.",
+                type(other),
+            )
+
+            return False
+        return True
+
     def _build_dataframes(self):
         """
         Function called when network is created to build component
@@ -522,11 +574,13 @@ def set_snapshots(
         """
         Set the snapshots/time steps and reindex all time-dependent data.
 
-        Snapshot weightings, typically representing the hourly length of each snapshot, is filled with the `default_snapshot_weighintgs` value,
-        or uses the timedelta of the snapshots if `weightings_from_timedelta` flag is True, and snapshots are of type `pd.DatetimeIndex`.
+        Snapshot weightings, typically representing the hourly length of each snapshot,
+        is filled with the `default_snapshot_weighintgs` value, or uses the timedelta
+        of the snapshots if `weightings_from_timedelta` flag is True, and snapshots are
+        of type `pd.DatetimeIndex`.
 
-        This will reindex all components time-dependent DataFrames (:role:`Network.pnl`); NaNs are filled
-        with the default value for that quantity.
+        This will reindex all components time-dependent DataFrames
+        (:role:`Network.pnl`). NaNs are filled with the default value for that quantity.
 
         Parameters
         ----------
@@ -552,6 +606,9 @@ def set_snapshots(
         else:
             self._snapshots = pd.Index(snapshots, name="snapshot")
 
+        if len(self._snapshots) == 0:
+            raise ValueError("Snapshots must not be empty.")
+
         self.snapshot_weightings = self.snapshot_weightings.reindex(
             self._snapshots, fill_value=default_snapshot_weightings
         )
@@ -576,11 +633,15 @@ def set_snapshots(
             pnl = self.pnl(component)
             attrs = self.components[component]["attrs"]
 
-            for k, default in attrs.default[attrs.varying].items():
+            for k in pnl.keys():
                 if pnl[k].empty:  # avoid expensive reindex operation
                     pnl[k].index = self._snapshots
+                elif k in attrs.default[attrs.varying]:
+                    pnl[k] = pnl[k].reindex(
+                        self._snapshots, fill_value=attrs.default[attrs.varying][k]
+                    )
                 else:
-                    pnl[k] = pnl[k].reindex(self._snapshots, fill_value=default)
+                    pnl[k] = pnl[k].reindex(self._snapshots)
 
         # NB: No need to rebind pnl to self, since haven't changed it
 
@@ -665,9 +726,8 @@ def set_investment_periods(self, periods):
             names = ["period", "timestep"]
             for component in self.all_components:
                 pnl = self.pnl(component)
-                attrs = self.components[component]["attrs"]
 
-                for k, default in attrs.default[attrs.varying].items():
+                for k in pnl.keys():
                     pnl[k] = pd.concat({p: pnl[k] for p in periods}, names=names)
                     pnl[k].index.name = "snapshot"
 
@@ -996,34 +1056,79 @@ def _retrieve_overridden_components(self):
 
     def copy(
         self,
-        with_time=True,
-        snapshots=None,
-        investment_periods=None,
-        ignore_standard_types=False,
+        snapshots: Collection = None,
+        investment_periods: Collection = None,
+        ignore_standard_types: bool = False,
+        with_time: bool = None,
     ):
         """
-        Returns a deep copy of the Network object with all components and time-
-        dependent data.
+        Returns a deep copy of Network object.
+
+        If only default arguments are passed, the copy will be created via
+        :func:`copy.deepcopy` and will contain all components and time-varying data.
+        For most networks this is the fastest way. However, if the network is very
+        large, it might be better to filter snapshots and investment periods to reduce
+        the size of the copy. In this case :func:`copy.deepcopy` is not used and only
+        the selected snapshots and investment periods are copied to a new object.
 
-        Returns
-        -------
-        network : pypsa.Network
 
         Parameters
         ----------
-        with_time : boolean, default True
-            Copy snapshots and time-varying network.component_names_t data too.
-        snapshots : list or index slice
-            A list of snapshots to copy, must be a subset of
-            network.snapshots, defaults to network.snapshots
+        snapshots : list or index slice , default self.snapshots
+            A list of snapshots to copy, must be a subset of network.snapshots. Pass
+            an empty list ignore all snapshots.
+        investment_periods : list or index slice, default self.investment_period_weightings.index
+            A list of investment periods to copy, must be a subset of network.investment_periods. Pass
         ignore_standard_types : boolean, default False
             Ignore the PyPSA standard types.
+        with_time : boolean, default True
+            Copy snapshots and time-varying network.component_names_t data too.
+
+            .. deprecated:: 0.29.0
+              Will be removed in a future version. Pass an empty list to 'snapshots'
+              instead.
+
+        Returns
+        -------
+        network : pypsa.Network
+            The copied network object.
 
         Examples
         --------
         >>> network_copy = network.copy()
 
         """
+
+        # Set default arguments
+        if snapshots is None:
+            snapshots = self.snapshots
+        if investment_periods is None:
+            investment_periods = self.investment_period_weightings.index
+
+        # Deprecation warnings
+        if with_time is not None:
+            warnings.warn(
+                "Argument 'with_time' is deprecated in 0.29 and will be "
+                "removed in a future version. Pass an empty list to 'snapshots' instead.",
+                DeprecationWarning,
+                stacklevel=2,
+            )
+            snapshots = []
+
+        if not isinstance(snapshots, (Collection)):
+            warnings.warn(
+                f"Argument 'snapshots' should be a collection, got {type(snapshots)}"
+                f"instead. This will raise an error in a future version.",
+                UserWarning,
+                stacklevel=2,
+            )
+
+        # Use copy.deepcopy if no arguments are passed
+        args = [snapshots, investment_periods, ignore_standard_types, with_time]
+        if all(arg is None or arg is False for arg in args):
+            return copy.deepcopy(self)
+
+        # Setup new network
         (
             override_components,
             override_component_attrs,
@@ -1035,10 +1140,9 @@ def copy(
             override_component_attrs=override_component_attrs,
         )
 
-        other_comps = sorted(self.all_components - {"Bus", "Carrier"})
-        for component in self.iterate_components(["Bus", "Carrier"] + other_comps):
-            df = component.df
-            # drop the standard types to avoid them being read in twice
+        # Copy components
+        for component in self.iterate_components(self.all_components):
+            # Drop the standard types to avoid them being read in twice
             if (
                 not ignore_standard_types
                 and component.name in self.standard_type_components
@@ -1046,29 +1150,49 @@ def copy(
                 df = component.df.drop(
                     network.components[component.name]["standard_types"].index
                 )
+            else:
+                df = component.df
 
             import_components_from_dataframe(network, df, component.name)
 
-        if with_time:
-            if snapshots is None:
-                snapshots = self.snapshots
-            if investment_periods is None:
-                investment_periods = self.investment_period_weightings.index
+        # Copy time-varying data, if given
+
+        if len(snapshots) > 0:
             network.set_snapshots(snapshots)
-            if not investment_periods.empty:
-                network.set_investment_periods(investment_periods)
+            # Apply time-varying data
             for component in self.iterate_components():
                 pnl = getattr(network, component.list_name + "_t")
                 for k in component.pnl.keys():
-                    pnl[k] = component.pnl[k].loc[snapshots].copy()
+                    try:
+                        pnl[k] = component.pnl[k].loc[snapshots].copy()
+                    except KeyError:
+                        pnl[k] = component.pnl[k].reindex(snapshots).copy()
+
+            # Apply investment periods
+            if not investment_periods.empty:
+                network.set_investment_periods(investment_periods)
+
+            # Add weightings
             network.snapshot_weightings = self.snapshot_weightings.loc[snapshots].copy()
             network.investment_period_weightings = (
                 self.investment_period_weightings.loc[investment_periods].copy()
             )
 
-        # catch all remaining attributes of network
-        for attr in ["name", "srid"]:
-            setattr(network, attr, getattr(self, attr))
+        # Catch all remaining attributes of network
+        for attr in [
+            "name",
+            "srid",
+            "_meta",
+            "_linearized_uc",
+            "_multi_invest",
+            "objective",
+            "objective_constant",
+            "now",
+        ]:
+            try:
+                setattr(network, attr, getattr(self, attr))
+            except AttributeError:
+                pass
 
         return network
 


Test patch:
diff --git a/test/conftest.py b/test/conftest.py
index 21e5e1bfa..0bd143284 100644
--- a/test/conftest.py
+++ b/test/conftest.py
@@ -112,6 +112,23 @@ def storage_hvdc_network():
     return pypsa.Network(csv_folder)
 
 
+@pytest.fixture(scope="module")
+def all_networks(
+    ac_dc_network,
+    ac_dc_network_r,
+    ac_dc_network_multiindexed,
+    ac_dc_network_shapes,
+    storage_hvdc_network,
+):
+    return [
+        ac_dc_network,
+        # ac_dc_network_r,
+        # ac_dc_network_multiindexed,
+        # ac_dc_network_shapes,
+        # storage_hvdc_network,
+    ]
+
+
 @pytest.fixture(scope="module")
 def pandapower_custom_network():
     net = pp.create_empty_network()
diff --git a/test/test_components.py b/test/test_components.py
index 9a4b7e9b9..13a0b6e3a 100644
--- a/test/test_components.py
+++ b/test/test_components.py
@@ -1,3 +1,6 @@
+import copy
+import sys
+
 import numpy as np
 import pytest
 
@@ -184,45 +187,52 @@ def test_madd_defaults(empty_network_5_buses):
     )
 
 
-def test_copy_default_behavior(ac_dc_network):
+@pytest.mark.skipif(
+    sys.platform == "win32",
+    reason="pd.equals fails on windows (https://stackoverflow.com/questions/62128721).",
+)
+def test_equality_behavior(all_networks):
     """
     GIVEN   the AC DC exemplary pypsa network.
 
-    WHEN    copying the network with timestamps
+    WHEN    comparing the network to itself
 
-    THEN    the copied network should have the same generators, loads
-    and timestamps.
+    THEN    the networks should be equal.
     """
-    snapshot = ac_dc_network.snapshots[2]
-    copied_network = ac_dc_network.copy()
-
-    loads = ac_dc_network.loads.index.tolist()
-    generators = ac_dc_network.generators.index.tolist()
-    copied_loads = copied_network.loads.index.tolist()
-    copied_generators = copied_network.generators.index.tolist()
+    for n in all_networks:
+        deep_copy = copy.deepcopy(n)
+        assert n == deep_copy
+        assert n is not deep_copy
 
-    assert loads == copied_loads
-    assert generators == copied_generators
-    assert not copied_network.snapshots.empty
-    assert snapshot in copied_network.snapshots
+        # TODO: Could add more property based tests here (hypothesis)
+        deep_copy.name = "new_name"
+        assert n != deep_copy
 
 
-def test_copy_deep_copy_behavior(ac_dc_network):
+@pytest.mark.skipif(
+    sys.platform == "win32",
+    reason="pd.equals fails on windows (https://stackoverflow.com/questions/62128721).",
+)
+def test_copy_default_behavior(all_networks):
     """
     GIVEN   the AC DC exemplary pypsa network.
 
-    WHEN    copying the network and changing a component
+    WHEN    copying the network with timestamps
 
-    THEN    the original network should have not changed.
+    THEN    the copied network should have the same generators, loads
+    and timestamps.
     """
-    copied_network = ac_dc_network.copy()
-
-    copied_network.loads.rename(index={"London": "Berlin"}, inplace=True)
-
-    assert ac_dc_network.loads.index[0] != copied_network.loads.index[0]
+    for network in all_networks:
+        network_copy = network.copy()
+        assert network == network_copy
+        assert network is not network_copy
 
 
-def test_copy_no_snapshot(ac_dc_network):
+@pytest.mark.skipif(
+    sys.platform == "win32",
+    reason="pd.equals fails on windows (https://stackoverflow.com/questions/62128721).",
+)
+def test_copy_snapshots(all_networks):
     """
     GIVEN   the AC DC exemplary pypsa network.
 
@@ -230,11 +240,13 @@ def test_copy_no_snapshot(ac_dc_network):
 
     THEN    the copied network should only have the current time index.
     """
-    snapshot = ac_dc_network.snapshots[2]
-    copied_network = ac_dc_network.copy(with_time=False, snapshots=snapshot)
+    for network in all_networks:
+        copied_network = network.copy(snapshots=[])
+        assert copied_network.snapshots.size == 1
 
-    assert copied_network.snapshots.size == 1
-    assert snapshot not in copied_network.snapshots
+        copied_network = network.copy(snapshots=network.snapshots[:5])
+        network.set_snapshots(network.snapshots[:5])
+        assert copied_network == network
 
 
 def test_add_network_static(ac_dc_network, empty_network_5_buses):