Repository: CS-SI/eodag (python)

Issue:
http asset filesize using GET method
Following #542, in `HTTPDownload` try getting asset filesize from header while downloading with GET method if the filesize could not be obtained with HEAD method (`total_size == 0`). Implement something like:
```py
if total_size == 0:
    asset_size = parse_header(asset_content_disposition)[-1].get("size", 0)
    progress_callback.total += asset_size
```

Pull request:
feat: http asset size from get method
fixes #564 

With descending priority, http asset size is now obtained from:
- HEAD header / Content-length
- HEAD header / content-disposition / size
- GET header / Content-length
- GET header / content-disposition / size

Patch:
diff --git a/eodag/plugins/download/http.py b/eodag/plugins/download/http.py
index 58fe8a89..9b9b66ba 100644
--- a/eodag/plugins/download/http.py
+++ b/eodag/plugins/download/http.py
@@ -252,6 +252,9 @@ class HTTPDownload(Download):
         assets_urls = [
             a["href"] for a in getattr(product, "assets", {}).values() if "href" in a
         ]
+        assets_values = [
+            a for a in getattr(product, "assets", {}).values() if "href" in a
+        ]
 
         if not assets_urls:
             raise NotAvailableError("No assets available for %s" % product)
@@ -271,55 +274,77 @@ class HTTPDownload(Download):
             "flatten_top_dirs", getattr(self.config, "flatten_top_dirs", False)
         )
 
-        total_size = sum(
-            [
-                # get total size using header[Content-length] of each asset
-                int(
-                    requests.head(
-                        asset_url, auth=auth, timeout=HTTP_REQ_TIMEOUT
-                    ).headers.get("Content-length", 0)
-                )
-                or int(  # alternative: get total size using header[content-disposition][size] of each asset
-                    parse_header(
-                        requests.head(
-                            asset_url, auth=auth, timeout=HTTP_REQ_TIMEOUT
-                        ).headers.get("content-disposition", "")
-                    )[-1].get("size", 0)
-                )
-                for asset_url in assets_urls
-                if not asset_url.startswith("file:")
-            ]
+        # get extra parameters to pass to the query
+        params = kwargs.pop("dl_url_params", None) or getattr(
+            self.config, "dl_url_params", {}
         )
+
+        total_size = 0
+        # loop for assets size & filename
+        for asset in assets_values:
+            if not asset["href"].startswith("file:"):
+                # HEAD request for size & filename
+                asset_headers = requests.head(
+                    asset["href"], auth=auth, timeout=HTTP_REQ_TIMEOUT
+                ).headers
+                header_content_disposition_dict = {}
+
+                if not asset.get("size", 0):
+                    # size from HEAD header / Content-length
+                    asset["size"] = int(asset_headers.get("Content-length", 0))
+
+                if not asset.get("size", 0) or not asset.get("filename", 0):
+                    # header content-disposition
+                    header_content_disposition_dict = parse_header(
+                        asset_headers.get("content-disposition", "")
+                    )[-1]
+                if not asset.get("size", 0):
+                    # size from HEAD header / content-disposition / size
+                    asset["size"] = int(header_content_disposition_dict.get("size", 0))
+                if not asset.get("filename", 0):
+                    # filename from HEAD header / content-disposition / size
+                    asset["filename"] = header_content_disposition_dict.get(
+                        "filename", None
+                    )
+
+                if not asset.get("size", 0):
+                    # GET request for size
+                    with requests.get(
+                        asset["href"],
+                        stream=True,
+                        auth=auth,
+                        params=params,
+                        timeout=DEFAULT_STREAM_REQUESTS_TIMEOUT,
+                    ) as stream:
+                        # size from GET header / Content-length
+                        asset["size"] = int(stream.headers.get("Content-length", 0))
+                        if not asset.get("size", 0):
+                            # size from GET header / content-disposition / size
+                            asset["size"] = int(
+                                parse_header(
+                                    stream.headers.get("content-disposition", "")
+                                )[-1].get("size", 0)
+                            )
+
+                total_size += asset["size"]
+
         progress_callback.reset(total=total_size)
         error_messages = set()
 
         local_assets_count = 0
 
-        for asset_url in assets_urls:
+        # loop for assets download
+        for asset in assets_values:
 
-            if asset_url.startswith("file:"):
-                logger.info(f"Local asset detected. Download skipped for {asset_url}")
+            if asset["href"].startswith("file:"):
+                logger.info(
+                    f"Local asset detected. Download skipped for {asset['href']}"
+                )
                 local_assets_count += 1
                 continue
 
-            # get asset filename from header
-            asset_content_disposition = requests.head(
-                asset_url, auth=auth, timeout=HTTP_REQ_TIMEOUT
-            ).headers.get("content-disposition", None)
-            if asset_content_disposition:
-                asset_filename = parse_header(asset_content_disposition)[-1].get(
-                    "filename", None
-                )
-            else:
-                asset_filename = None
-
-            # get extra parameters to pass to the query
-            params = kwargs.pop("dl_url_params", None) or getattr(
-                self.config, "dl_url_params", {}
-            )
-
             with requests.get(
-                asset_url,
+                asset["href"],
                 stream=True,
                 auth=auth,
                 params=params,
@@ -343,28 +368,28 @@ class HTTPDownload(Download):
                         )
                     else:
                         logger.warning("Unexpected error: %s" % e)
-                        logger.warning("Skipping %s" % asset_url)
+                        logger.warning("Skipping %s" % asset["href"])
                     error_messages.add(str(e))
                 else:
-                    asset_rel_path = urlparse(asset_url).path.strip("/")
+                    asset_rel_path = urlparse(asset["href"]).path.strip("/")
                     asset_rel_dir = os.path.dirname(asset_rel_path)
 
-                    if asset_filename is None:
-                        # try getting content-disposition header from GET if not in HEAD result
-                        asset_content_disposition = stream.headers.get(
+                    if not asset.get("filename", None):
+                        # try getting filename in GET header if was not found in HEAD result
+                        asset_content_disposition_dict = stream.headers.get(
                             "content-disposition", None
                         )
-                        if asset_content_disposition:
-                            asset_filename = parse_header(asset_content_disposition)[
-                                -1
-                            ].get("filename", None)
+                        if asset_content_disposition_dict:
+                            asset["filename"] = parse_header(
+                                asset_content_disposition_dict
+                            )[-1].get("filename", None)
 
-                    if asset_filename is None:
+                    if not asset.get("filename", None):
                         # default filename extracted from path
-                        asset_filename = os.path.basename(asset_rel_path)
+                        asset["filename"] = os.path.basename(asset_rel_path)
 
                     asset_abs_path = os.path.join(
-                        fs_dir_path, asset_rel_dir, asset_filename
+                        fs_dir_path, asset_rel_dir, asset["filename"]
                     )
                     asset_abs_path_dir = os.path.dirname(asset_abs_path)
                     if not os.path.isdir(asset_abs_path_dir):


Test patch:
diff --git a/tests/units/test_download_plugins.py b/tests/units/test_download_plugins.py
index d31a976b..f37a1b01 100644
--- a/tests/units/test_download_plugins.py
+++ b/tests/units/test_download_plugins.py
@@ -239,6 +239,73 @@ class TestDownloadPluginHttp(BaseDownloadPluginTest):
             )
         )
 
+    @mock.patch("eodag.utils.ProgressCallback.reset", autospec=True)
+    @mock.patch("eodag.plugins.download.http.requests.head", autospec=True)
+    @mock.patch("eodag.plugins.download.http.requests.get", autospec=True)
+    def test_plugins_download_http_assets_size(
+        self, mock_requests_get, mock_requests_head, mock_progress_callback_reset
+    ):
+        """HTTPDownload.download() must get assets sizes"""
+
+        plugin = self.get_download_plugin(self.product)
+        self.product.location = self.product.remote_location = "http://somewhere"
+        self.product.assets = {
+            "foo": {"href": "http://somewhere/a"},
+            "bar": {"href": "http://somewhere/b"},
+        }
+
+        mock_requests_head.return_value.headers = {
+            "Content-length": "1",
+            "content-disposition": '; size = "2"',
+        }
+        mock_requests_get.return_value.__enter__.return_value.headers = {
+            "Content-length": "3",
+            "content-disposition": '; size = "4"',
+        }
+
+        # size from HEAD / Content-length
+        with TemporaryDirectory() as temp_dir:
+            plugin.download(self.product, outputs_prefix=temp_dir)
+        mock_progress_callback_reset.assert_called_once_with(mock.ANY, total=1 + 1)
+
+        # size from HEAD / content-disposition
+        mock_requests_head.return_value.headers.pop("Content-length")
+        mock_progress_callback_reset.reset_mock()
+        self.product.location = "http://somewhere"
+        self.product.assets = {
+            "foo": {"href": "http://somewhere/a"},
+            "bar": {"href": "http://somewhere/b"},
+        }
+        with TemporaryDirectory() as temp_dir:
+            plugin.download(self.product, outputs_prefix=temp_dir)
+        mock_progress_callback_reset.assert_called_once_with(mock.ANY, total=2 + 2)
+
+        # size from GET / Content-length
+        mock_requests_head.return_value.headers.pop("content-disposition")
+        mock_progress_callback_reset.reset_mock()
+        self.product.location = "http://somewhere"
+        self.product.assets = {
+            "foo": {"href": "http://somewhere/a"},
+            "bar": {"href": "http://somewhere/b"},
+        }
+        with TemporaryDirectory() as temp_dir:
+            plugin.download(self.product, outputs_prefix=temp_dir)
+        mock_progress_callback_reset.assert_called_once_with(mock.ANY, total=3 + 3)
+
+        # size from GET / content-disposition
+        mock_requests_get.return_value.__enter__.return_value.headers.pop(
+            "Content-length"
+        )
+        mock_progress_callback_reset.reset_mock()
+        self.product.location = "http://somewhere"
+        self.product.assets = {
+            "foo": {"href": "http://somewhere/a"},
+            "bar": {"href": "http://somewhere/b"},
+        }
+        with TemporaryDirectory() as temp_dir:
+            plugin.download(self.product, outputs_prefix=temp_dir)
+        mock_progress_callback_reset.assert_called_once_with(mock.ANY, total=4 + 4)
+
     def test_plugins_download_http_one_local_asset(
         self,
     ):