Repository: CS-SI/eodag
Pull request #664 (2023-03-02 14:44:20)

feat: customized and faster deepcopy
Customized and faster deepcopy inspired by https://stackoverflow.com/a/45858907

```py
import copy
import timeit
from eodag.config import load_stac_config
from eodag.utils import deepcopy

stac_config = load_stac_config()

print("New deepcopy: %s ms" % timeit.timeit("deepcopy(stac_config)", globals=locals(), number=1000))
print("copy.deepcopy: %s ms" % timeit.timeit("copy.deepcopy(stac_config)", globals=locals(), number=1000))
```
Results in:
```
New deepcopy: 0.21507359799943515 ms
copy.deepcopy: 7.414625944002182 ms
```


github-actions[bot]: ## Unit Test Results
    2 files  ±0      2 suites  ±0   4m 42s [:stopwatch:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "duration of all tests") - 1m 25s
373 tests +1  370 [:heavy_check_mark:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "passed tests") +1  2 [:zzz:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "skipped / disabled tests")  - 1  1 [:x:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "failed tests") +1 
746 runs  +2  741 [:heavy_check_mark:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "passed tests") +3  4 [:zzz:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "skipped / disabled tests")  - 2  1 [:x:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "failed tests") +1 

For more details on these failures, see [this check](https://github.com/CS-SI/eodag/runs/11718739952).

Results for commit a2e0fd0c. ± Comparison against base commit 6b923611.

[test-results]:data:application/gzip;base64,H4sIAFa3AGQC/02MQQ6DIBQFr2JYdwF8ItLLNF+EhFSlQViZ3r2gQl3OvLzZiXWz2ciz44+ObMnFBlMKGJ1fCw5F5CmWESRUem1J60PRv3q7z5U4hUU3Z8GaMCH4kE35hLSWphT9BTUpBWvmLIrKt+DB9572y+JiBoLcUDtRjciBccBRoRoUHQ0DBRY0GiN6JST5/gBO8MXRBQEAAA==

github-actions[bot]: <strong> </strong>

| File | Coverage |   |
| - | :-: | :-: |
| **All files** | `88%` | :white_check_mark: |
| eodag/config.py | `88%` | :white_check_mark: |
| eodag/api/core.py | `92%` | :white_check_mark: |
| eodag/api/product/metadata\_mapping.py | `89%` | :white_check_mark: |
| eodag/plugins/apis/usgs.py | `81%` | :white_check_mark: |
| eodag/rest/stac.py | `79%` | :white_check_mark: |
| eodag/utils/\_\_init\_\_.py | `90%` | :white_check_mark: |
| tests/units/test\_core.py | `98%` | :white_check_mark: |
| tests/units/test\_utils.py | `96%` | :white_check_mark: |

<strong> </strong>

| File | Coverage |   |
| - | :-: | :-: |
| **All files** | `26%` | :x: |
| eodag/config.py | `88%` | :white_check_mark: |
| eodag/api/core.py | `92%` | :white_check_mark: |
| eodag/api/product/metadata\_mapping.py | `89%` | :white_check_mark: |
| eodag/plugins/apis/usgs.py | `81%` | :white_check_mark: |
| eodag/rest/stac.py | `79%` | :white_check_mark: |
| eodag/utils/\_\_init\_\_.py | `90%` | :white_check_mark: |
| tests/units/test\_core.py | `98%` | :white_check_mark: |
| tests/units/test\_utils.py | `96%` | :white_check_mark: |
| eodag/config.py | `82%` | :white_check_mark: |
| eodag/api/core.py | `30%` | :x: |
| eodag/api/product/metadata\_mapping.py | `9%` | :x: |
| eodag/plugins/apis/usgs.py | `12%` | :x: |
| eodag/utils/\_\_init\_\_.py | `60%` | :x: |
| tests/units/test\_core.py | `16%` | :x: |
| tests/units/test\_utils.py | `16%` | :x: |

_Minimum allowed coverage is `70%`_

<p align="right">Generated by :monkey: cobertura-action against a2e0fd0caa23123ab9a9890be1393f3caee46947 </p>

Patch:
diff --git a/eodag/api/core.py b/eodag/api/core.py
index a08100d5..3c348c82 100644
--- a/eodag/api/core.py
+++ b/eodag/api/core.py
@@ -19,7 +19,6 @@ import logging
 import os
 import re
 import shutil
-from copy import deepcopy
 from operator import itemgetter
 
 import geojson
@@ -50,6 +49,7 @@ from eodag.utils import (
     GENERIC_PRODUCT_TYPE,
     MockResponse,
     _deprecated,
+    deepcopy,
     get_geometry_from_various,
     makedirs,
     obj_md5sum,
diff --git a/eodag/api/product/metadata_mapping.py b/eodag/api/product/metadata_mapping.py
index 4d730512..14c3ece4 100644
--- a/eodag/api/product/metadata_mapping.py
+++ b/eodag/api/product/metadata_mapping.py
@@ -18,7 +18,6 @@
 import ast
 import logging
 import re
-from copy import deepcopy
 from datetime import datetime, timedelta
 from functools import partial
 from string import Formatter
@@ -37,6 +36,7 @@ from shapely.ops import transform
 from eodag.utils import (
     DEFAULT_PROJ,
     cached_parse,
+    deepcopy,
     get_timestamp,
     items_recursive_apply,
     nested_pairs2dict,
diff --git a/eodag/config.py b/eodag/config.py
index 9339a3fd..bd696993 100644
--- a/eodag/config.py
+++ b/eodag/config.py
@@ -19,7 +19,6 @@ import json
 import logging
 import os
 import tempfile
-from copy import deepcopy
 
 import requests
 import yaml
@@ -29,6 +28,7 @@ from pkg_resources import resource_filename
 
 from eodag.utils import (
     USER_AGENT,
+    deepcopy,
     dict_items_recursive_apply,
     merge_mappings,
     slugify,
diff --git a/eodag/plugins/apis/usgs.py b/eodag/plugins/apis/usgs.py
index 3293a073..26fb40a7 100644
--- a/eodag/plugins/apis/usgs.py
+++ b/eodag/plugins/apis/usgs.py
@@ -15,7 +15,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import copy
 import logging
 import shutil
 import tarfile
@@ -42,6 +41,7 @@ from eodag.utils import (
     GENERIC_PRODUCT_TYPE,
     USER_AGENT,
     ProgressCallback,
+    deepcopy,
     format_dict_items,
     path_to_uri,
 )
@@ -167,7 +167,7 @@ class UsgsApi(Download, Api):
             # added by the provider mapping that is not in the default metadata.
             # A deepcopy is done to prevent self.config.metadata_mapping from being modified when metas[metadata]
             # is a list and is modified
-            metas.update(copy.deepcopy(self.config.metadata_mapping))
+            metas.update(deepcopy(self.config.metadata_mapping))
             # common_jsonpath usage to optimize jsonpath build process
             mtd_cfg_as_jsonpath_options = {}
             if hasattr(self.config, "common_metadata_mapping_path"):
diff --git a/eodag/rest/stac.py b/eodag/rest/stac.py
index d338e876..67f6bf64 100644
--- a/eodag/rest/stac.py
+++ b/eodag/rest/stac.py
@@ -15,7 +15,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import copy
 import datetime
 import logging
 import os
@@ -35,6 +34,7 @@ from eodag.api.product.metadata_mapping import (
     get_metadata_path,
 )
 from eodag.utils import (
+    deepcopy,
     dict_items_recursive_apply,
     format_dict_items,
     jsonpath_parse_dict_items,
@@ -136,12 +136,10 @@ class StacCommon(object):
         :returns: STAC extension as dictionnary
         :rtype: dict
         """
-        extension_model = (
-            copy.deepcopy(stac_config).get("extensions", {}).get(extension, {})
-        )
+        extension_model = deepcopy(stac_config).get("extensions", {}).get(extension, {})
 
         # parse f-strings
-        format_args = copy.deepcopy(stac_config)
+        format_args = deepcopy(stac_config)
         format_args["extension"] = {
             "url": url,
             "properties": kwargs.get("properties", {}),
@@ -243,7 +241,7 @@ class StacItem(StacCommon):
                     )
 
             # parse f-strings
-            format_args = copy.deepcopy(self.stac_config)
+            format_args = deepcopy(self.stac_config)
             format_args["catalog"] = catalog
             format_args["item"] = product_item
             product_item = format_dict_items(product_item, **format_args)
@@ -266,7 +264,7 @@ class StacItem(StacCommon):
         :returns: Items dictionnary
         :rtype: dict
         """
-        items_model = copy.deepcopy(self.stac_config["items"])
+        items_model = deepcopy(self.stac_config["items"])
 
         search_results.numberMatched = search_results.properties["totalResults"]
         search_results.numberReturned = len(search_results)
@@ -306,7 +304,7 @@ class StacItem(StacCommon):
             items_model, {"search_results": search_results.__dict__}
         )
         # parse f-strings
-        format_args = copy.deepcopy(self.stac_config)
+        format_args = deepcopy(self.stac_config)
         format_args["catalog"] = catalog
         items = format_dict_items(items, **format_args)
 
@@ -359,7 +357,7 @@ class StacItem(StacCommon):
                 )
             )
 
-        result_item_model = copy.deepcopy(item_model)
+        result_item_model = deepcopy(item_model)
         result_item_model["stac_extensions"] = list(
             self.stac_config["stac_extensions"].values()
         )
@@ -405,7 +403,7 @@ class StacItem(StacCommon):
         :returns: Filtered item model
         :rtype: dict
         """
-        all_extensions_dict = copy.deepcopy(self.stac_config["stac_extensions"])
+        all_extensions_dict = deepcopy(self.stac_config["stac_extensions"])
         # parse f-strings with root
         all_extensions_dict = format_dict_items(
             all_extensions_dict, **{"catalog": {"root": self.root}}
@@ -456,7 +454,7 @@ class StacItem(StacCommon):
             item_model, {"product": product.__dict__}
         )
         # parse f-strings
-        format_args = copy.deepcopy(self.stac_config)
+        format_args = deepcopy(self.stac_config)
         # format_args["collection"] = dict(catalog.as_dict(), **{"url": catalog.url})
         format_args["catalog"] = dict(
             catalog.as_dict(), **{"url": catalog.url, "root": catalog.root}
@@ -538,7 +536,7 @@ class StacCollection(StacCommon):
         :returns: STAC collection dicts list
         :rtype: list
         """
-        collection_model = copy.deepcopy(self.stac_config["collection"])
+        collection_model = deepcopy(self.stac_config["collection"])
 
         product_types = self.__get_product_types(filters)
 
@@ -569,7 +567,7 @@ class StacCollection(StacCommon):
                 },
             )
             # parse f-strings
-            format_args = copy.deepcopy(self.stac_config)
+            format_args = deepcopy(self.stac_config)
             format_args["collection"] = dict(
                 product_type_collection, **{"url": self.url, "root": self.root}
             )
@@ -589,7 +587,7 @@ class StacCollection(StacCommon):
         :returns: Collections dictionnary
         :rtype: dict
         """
-        collections = copy.deepcopy(self.stac_config["collections"])
+        collections = deepcopy(self.stac_config["collections"])
         collections["collections"] = self.__get_collection_list(filters)
 
         collections["links"] += [
@@ -672,7 +670,7 @@ class StacCatalog(StacCommon):
         self.data = {}
         self.children = []
 
-        self.catalog_config = copy.deepcopy(stac_config["catalog"])
+        self.catalog_config = deepcopy(stac_config["catalog"])
 
         self.__update_data_from_catalog_config({"model": {}})
 
@@ -698,7 +696,7 @@ class StacCatalog(StacCommon):
 
         # parse f-strings
         # defaultdict usage will return "" for missing keys in format_args
-        format_args = copy.deepcopy(self.stac_config)
+        format_args = deepcopy(self.stac_config)
         format_args["catalog"] = defaultdict(
             str, dict(model, **{"root": self.root, "url": self.url})
         )
@@ -735,9 +733,9 @@ class StacCatalog(StacCommon):
             root=self.root,
         ).get_collection_by_id(product_type)
 
-        cat_model = copy.deepcopy(self.stac_config["catalogs"]["product_type"]["model"])
+        cat_model = deepcopy(self.stac_config["catalogs"]["product_type"]["model"])
         # parse f-strings
-        format_args = copy.deepcopy(self.stac_config)
+        format_args = deepcopy(self.stac_config)
         format_args["catalog"] = defaultdict(str, **self.data)
         format_args["collection"] = collection
         try:
@@ -815,7 +813,7 @@ class StacCatalog(StacCommon):
             ]
         )
 
-        catalog_model = copy.deepcopy(self.stac_config["catalogs"]["year"]["model"])
+        catalog_model = deepcopy(self.stac_config["catalogs"]["year"]["model"])
 
         parsed_dict = self.set_stac_date(datetime_min, datetime_max, catalog_model)
 
@@ -846,7 +844,7 @@ class StacCatalog(StacCommon):
             ]
         )
 
-        catalog_model = copy.deepcopy(self.stac_config["catalogs"]["month"]["model"])
+        catalog_model = deepcopy(self.stac_config["catalogs"]["month"]["model"])
 
         parsed_dict = self.set_stac_date(datetime_min, datetime_max, catalog_model)
 
@@ -878,7 +876,7 @@ class StacCatalog(StacCommon):
             ]
         )
 
-        catalog_model = copy.deepcopy(self.stac_config["catalogs"]["day"]["model"])
+        catalog_model = deepcopy(self.stac_config["catalogs"]["day"]["model"])
 
         parsed_dict = self.set_stac_date(datetime_min, datetime_max, catalog_model)
 
@@ -925,7 +923,7 @@ class StacCatalog(StacCommon):
         :rtype: dict
         """
         # parse f-strings
-        format_args = copy.deepcopy(self.stac_config)
+        format_args = deepcopy(self.stac_config)
         format_args["catalog"] = defaultdict(str, **self.data)
         format_args["date"] = defaultdict(
             str,
@@ -968,9 +966,9 @@ class StacCatalog(StacCommon):
         :returns: Updated catalog
         :rtype: dict
         """
-        cat_model = copy.deepcopy(self.stac_config["catalogs"]["cloud_cover"]["model"])
+        cat_model = deepcopy(self.stac_config["catalogs"]["cloud_cover"]["model"])
         # parse f-strings
-        format_args = copy.deepcopy(self.stac_config)
+        format_args = deepcopy(self.stac_config)
         format_args["catalog"] = defaultdict(str, **self.data)
         format_args["cloud_cover"] = cloud_cover
         parsed_dict = format_dict_items(cat_model, **format_args)
@@ -1064,9 +1062,9 @@ class StacCatalog(StacCommon):
 
         geom = unary_union(geom_hits)
 
-        cat_model = copy.deepcopy(self.stac_config["catalogs"]["country"]["model"])
+        cat_model = deepcopy(self.stac_config["catalogs"]["country"]["model"])
         # parse f-strings
-        format_args = copy.deepcopy(self.stac_config)
+        format_args = deepcopy(self.stac_config)
         format_args["catalog"] = defaultdict(str, **self.data)
         format_args["feature"] = defaultdict(str, {"geometry": geom, "id": location})
         parsed_dict = format_dict_items(cat_model, **format_args)
@@ -1086,7 +1084,7 @@ class StacCatalog(StacCommon):
         """
         user_config_locations_list = self.eodag_api.locations_config
 
-        locations_config_model = copy.deepcopy(self.stac_config["locations_catalogs"])
+        locations_config_model = deepcopy(self.stac_config["locations_catalogs"])
 
         locations_config = {}
         for loc in user_config_locations_list:
@@ -1118,7 +1116,7 @@ class StacCatalog(StacCommon):
         locations_config = self.build_locations_config()
 
         self.stac_config["catalogs"] = dict(
-            copy.deepcopy(self.stac_config["catalogs"]), **locations_config
+            deepcopy(self.stac_config["catalogs"]), **locations_config
         )
 
         if len(catalogs) == 0:
@@ -1126,11 +1124,10 @@ class StacCatalog(StacCommon):
             self.__update_data_from_catalog_config(
                 {
                     "model": dict(
-                        copy.deepcopy(self.stac_config["landing_page"]),
+                        deepcopy(self.stac_config["landing_page"]),
                         **{"provider": self.provider},
                     )
                 }
-                # {"model": copy.deepcopy(self.stac_config["landing_page"])}
             )
 
             # build children : product_types
@@ -1153,7 +1150,7 @@ class StacCatalog(StacCommon):
         else:
             # use product_types_list as base for building nested catalogs
             self.__update_data_from_catalog_config(
-                copy.deepcopy(self.stac_config["catalogs"]["product_types_list"])
+                deepcopy(self.stac_config["catalogs"]["product_types_list"])
             )
 
         for idx, cat in enumerate(catalogs):
@@ -1236,7 +1233,7 @@ class StacCatalog(StacCommon):
                     raise ValidationError(
                         "Bad settings for %s in stac_config catalogs" % cat
                     )
-                cat_config = copy.deepcopy(self.stac_config["catalogs"][cat_key])
+                cat_config = deepcopy(self.stac_config["catalogs"][cat_key])
                 # update data
                 self.__update_data_from_catalog_config(cat_config)
 
diff --git a/eodag/utils/__init__.py b/eodag/utils/__init__.py
index 97c1e9c5..407aba2c 100644
--- a/eodag/utils/__init__.py
+++ b/eodag/utils/__init__.py
@@ -21,7 +21,6 @@ Everything that does not fit into one of the specialised categories of utilities
 this package should go here
 """
 import ast
-import copy
 import datetime
 import errno
 import functools
@@ -755,7 +754,7 @@ def dict_items_recursive_apply(config_dict, apply_method, **apply_method_paramet
     :returns: Updated dict
     :rtype: dict
     """
-    result_dict = copy.deepcopy(config_dict)
+    result_dict = deepcopy(config_dict)
     for dict_k, dict_v in result_dict.items():
         if isinstance(dict_v, dict):
             result_dict[dict_k] = dict_items_recursive_apply(
@@ -791,7 +790,7 @@ def list_items_recursive_apply(config_list, apply_method, **apply_method_paramet
     :returns: Updated list
     :rtype: list
     """
-    result_list = copy.deepcopy(config_list)
+    result_list = deepcopy(config_list)
     for list_idx, list_v in enumerate(result_list):
         if isinstance(list_v, dict):
             result_list[list_idx] = dict_items_recursive_apply(
@@ -848,7 +847,7 @@ def dict_items_recursive_sort(config_dict):
     :returns: Updated dict
     :rtype: dict
     """
-    result_dict = copy.deepcopy(config_dict)
+    result_dict = deepcopy(config_dict)
     for dict_k, dict_v in result_dict.items():
         if isinstance(dict_v, dict):
             result_dict[dict_k] = dict_items_recursive_sort(dict_v)
@@ -871,7 +870,7 @@ def list_items_recursive_sort(config_list):
     :returns: Updated list
     :rtype: list
     """
-    result_list = copy.deepcopy(config_list)
+    result_list = deepcopy(config_list)
     for list_idx, list_v in enumerate(result_list):
         if isinstance(list_v, dict):
             result_list[list_idx] = dict_items_recursive_sort(list_v)
@@ -1200,3 +1199,40 @@ def flatten_top_directories(nested_dir_root, common_subdirs_path=None):
         shutil.copytree(common_subdirs_path, tmp_path)
         shutil.rmtree(nested_dir_root)
         shutil.move(tmp_path, nested_dir_root)
+
+
+def deepcopy(sth):
+    """Customized and faster deepcopy inspired by https://stackoverflow.com/a/45858907
+    `_copy_list` and `_copy_dict` available for the moment
+
+    :param sth: Object to copy
+    :type sth: Any
+    :returns: Copied object
+    :rtype: Any
+    """
+    _dispatcher = {}
+
+    def _copy_list(input_list, dispatch):
+        ret = input_list.copy()
+        for idx, item in enumerate(ret):
+            cp = dispatch.get(type(item))
+            if cp is not None:
+                ret[idx] = cp(item, dispatch)
+        return ret
+
+    def _copy_dict(input_dict, dispatch):
+        ret = input_dict.copy()
+        for key, value in ret.items():
+            cp = dispatch.get(type(value))
+            if cp is not None:
+                ret[key] = cp(value, dispatch)
+        return ret
+
+    _dispatcher[list] = _copy_list
+    _dispatcher[dict] = _copy_dict
+
+    cp = _dispatcher.get(type(sth))
+    if cp is None:
+        return sth
+    else:
+        return cp(sth, _dispatcher)


Test patch:
diff --git a/tests/context.py b/tests/context.py
index c2c23880..63f4f7f7 100644
--- a/tests/context.py
+++ b/tests/context.py
@@ -78,6 +78,7 @@ from eodag.utils import (
     urlsplit,
     GENERIC_PRODUCT_TYPE,
     flatten_top_directories,
+    deepcopy,
 )
 from eodag.utils.exceptions import (
     AddressNotFound,
diff --git a/tests/units/test_core.py b/tests/units/test_core.py
index e10c4fc9..16c349f1 100644
--- a/tests/units/test_core.py
+++ b/tests/units/test_core.py
@@ -16,6 +16,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import copy
 import glob
 import json
 import os
@@ -23,7 +24,6 @@ import re
 import shutil
 import unittest
 import uuid
-from copy import deepcopy
 from pathlib import Path
 from tempfile import TemporaryDirectory
 
@@ -1127,7 +1127,7 @@ class TestCoreSearch(TestCoreBase):
         cls.search_results = SearchResult.from_geojson(search_results_geojson)
         cls.search_results_size = len(cls.search_results)
         # Change the id of these products, to emulate different products
-        search_results_data_2 = deepcopy(cls.search_results.data)
+        search_results_data_2 = copy.deepcopy(cls.search_results.data)
         search_results_data_2[0].properties["id"] = "a"
         search_results_data_2[1].properties["id"] = "b"
         cls.search_results_2 = SearchResult(search_results_data_2)
diff --git a/tests/units/test_utils.py b/tests/units/test_utils.py
index 90697d31..e260061e 100644
--- a/tests/units/test_utils.py
+++ b/tests/units/test_utils.py
@@ -16,6 +16,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import copy
 import os
 import sys
 import unittest
@@ -28,6 +29,7 @@ from tempfile import TemporaryDirectory
 from tests.context import (
     DownloadedCallback,
     ProgressCallback,
+    deepcopy,
     flatten_top_directories,
     get_bucket_name_and_prefix,
     get_timestamp,
@@ -276,3 +278,15 @@ class TestUtils(unittest.TestCase):
             self.assertIn(Path(nested_dir_root) / "b" / "c2", dir_content)
             self.assertIn(Path(nested_dir_root) / "b" / "c2" / "bar", dir_content)
             self.assertIn(Path(nested_dir_root) / "b" / "c2" / "baz", dir_content)
+
+    def test_deepcopy(self):
+        """deepcopy must to a recursice copy of the given object"""
+        original = {"a": [{"b": [0, 1, 2]}]}
+        shallow_copied = copy.copy(original)
+        deep_copied = deepcopy(original)
+        # change original
+        original["a"][0]["b"][0] = 5
+        # shallow copy also changed
+        self.assertEqual(shallow_copied["a"][0]["b"][0], 5)
+        # deep copy did not change
+        self.assertEqual(deep_copied["a"][0]["b"][0], 0)