Repository: CS-SI/eodag
Pull request #745 (2023-06-21 14:48:38)

fix: sanitize downloaded assets paths
Sanitizes downloaded assets paths, and fixes issue when downloading a `hydroweb_next` product on windows:
```py
from eodag import EODataAccessGateway

dag = EODataAccessGateway()
search_results = dag.search_all(
    productType="SWOT_L2_HR_LAKESP_PRIOR_SAMPLE_V1_2",
)
downloaded_paths = dag.download_all(search_results)
```
```
URN:FEATURE:DATA:hysope2:2630a915-c732-3730-be8b-d21ef954d411:V1:   0%|          | 0.00/3.35M [00:00<?, ?B/s][A2023-05-25 17:05:45,145 eodag.plugins.download.base      [WARNING ] (base             ) A problem occurred during download of product: EOProduct(id=URN:FEATURE:DATA:hysope2:2630a915-c732-3730-be8b-d21ef954d411:V1, provider=hydroweb_next). Skipping it
2023-05-25 17:05:45,147 eodag.plugins.download.base      [DEBUG   ] (base             )
Traceback (most recent call last):
  File "C:\dvlt\python\Python-3.8.10\lib\site-packages\eodag\plugins\download\base.py", line 453, in download_all
    product.download(
  File "C:\dvlt\python\Python-3.8.10\lib\site-packages\eodag\api\product\_product.py", line 325, in download
    fs_path = self.downloader.download(
  File "C:\dvlt\python\Python-3.8.10\lib\site-packages\eodag\plugins\download\http.py", line 344, in download
    fs_path = self._download_assets(
  File "C:\dvlt\python\Python-3.8.10\lib\site-packages\eodag\plugins\download\http.py", line 668, in _download_assets
    os.makedirs(asset_abs_path_dir)
  File "C:\dvlt\python\Python-3.8.10\lib\os.py", line 213, in makedirs
    makedirs(head, exist_ok=exist_ok)
  File "C:\dvlt\python\Python-3.8.10\lib\os.py", line 223, in makedirs
    mkdir(name, mode)
OSError: [WinError 123] La syntaxe du nom de fichier, de répertoire ou de volume est incorrecte: 'C:\\Users\\Someone\\AppData\\Local\\Temp\\URN_FEATURE_DATA_hysope2_2630a915-c732-3730-be8b-d21ef954d411_V1-URN_FEATURE_DATA_hysope2_2630a915-c732-3730-be8b-d21ef954d411_V1\\api/v1/rs-catalog/downloads/URN:FEATURE:DATA:hysope2:2630a915-c732-3730-be8b-d21ef954d411:V1'
```

github-actions[bot]: ## Test Results
       4 files  ±0         4 suites  ±0   4m 40s [:stopwatch:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v2.8.0/README.md#the-symbols "duration of all tests") +15s
   388 tests ±0     386 [:heavy_check_mark:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v2.8.0/README.md#the-symbols "passed tests") ±0    2 [:zzz:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v2.8.0/README.md#the-symbols "skipped / disabled tests") ±0  0 [:x:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v2.8.0/README.md#the-symbols "failed tests") ±0 
1 552 runs  ±0  1 496 [:heavy_check_mark:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v2.8.0/README.md#the-symbols "passed tests") ±0  56 [:zzz:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v2.8.0/README.md#the-symbols "skipped / disabled tests") ±0  0 [:x:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v2.8.0/README.md#the-symbols "failed tests") ±0 

Results for commit 569787ae. ± Comparison against base commit 274b6b90.

[test-results]:data:application/gzip;base64,H4sIAAkPk2QC/02Myw6DIBQFf8Ww7oKLl1d/xhDEhFSlQViZ/nvBKnU5c3JmJ5Of3UaeHT46smWfGow5muTDWpApWkSZUh17pS4atmztocRfvfy7fpqYjJ+LaInBxRjiaWJeaxM4ZyddTUAtmvo1eRO35sH3pA3L4lMBwoWWShqHyAE5asYAkFKA3gorNQgnmDOjIp8vA9pvxQgBAAA=

github-actions[bot]: <strong>Code Coverage (Windows)</strong>

| File | Coverage |   |
| - | :-: | :-: |
| **All files** | `77%` | :white_check_mark: |

_Minimum allowed coverage is `70%`_

<p align="right">Generated by :monkey: cobertura-action against 569787ae445145492211400113c6c7916e62ead8 </p>

Patch:
diff --git a/eodag/plugins/download/http.py b/eodag/plugins/download/http.py
index ff52491e..4a5a20ab 100644
--- a/eodag/plugins/download/http.py
+++ b/eodag/plugins/download/http.py
@@ -46,6 +46,7 @@ from eodag.utils import (
     flatten_top_directories,
     parse_header,
     path_to_uri,
+    sanitize,
     uri_to_path,
 )
 from eodag.utils.exceptions import (
@@ -643,7 +644,13 @@ class HTTPDownload(Download):
                         logger.warning("Skipping %s" % asset["href"])
                     error_messages.add(str(e))
                 else:
-                    asset_rel_path = urlparse(asset["href"]).path.strip("/")
+                    asset_rel_path_parts = (
+                        urlparse(asset["href"]).path.strip("/").split("/")
+                    )
+                    asset_rel_path_parts_sanitized = [
+                        sanitize(part) for part in asset_rel_path_parts
+                    ]
+                    asset_rel_path = os.path.join(*asset_rel_path_parts_sanitized)
                     asset_rel_dir = os.path.dirname(asset_rel_path)
 
                     if not asset.get("filename", None):


Test patch:
diff --git a/tests/units/test_download_plugins.py b/tests/units/test_download_plugins.py
index 902a8e56..0755ad55 100644
--- a/tests/units/test_download_plugins.py
+++ b/tests/units/test_download_plugins.py
@@ -182,7 +182,7 @@ class TestDownloadPluginHttp(BaseDownloadPluginTest):
         self.product.location = self.product.remote_location = "http://somewhere"
         self.product.properties["id"] = "someproduct"
         self.product.assets = {
-            "foo": {"href": "http://somewhere/something?foo=bar#baz"}
+            "foo": {"href": "http://somewhere/mal:for;matted/something?foo=bar#baz"}
         }
         mock_requests_get.return_value.__enter__.return_value.headers = {
             "content-disposition": ""
@@ -194,7 +194,11 @@ class TestDownloadPluginHttp(BaseDownloadPluginTest):
         self.assertEqual(path, os.path.join(self.output_dir, "dummy_product"))
         self.assertTrue(os.path.isdir(path))
         self.assertTrue(
-            os.path.isfile(os.path.join(self.output_dir, "dummy_product", "something"))
+            os.path.isfile(
+                os.path.join(
+                    self.output_dir, "dummy_product", "mal_for_matted", "something"
+                )
+            )
         )
 
         # Check if the GET request has been called for both size request and download request