Repository: CS-SI/eodag (python)

Issue:
staging status in server mode download 
When downloading an OFFLINE product in server mode, use min values for `wait` and `time` and return a 202 status until product is available

Pull request:
feat[server]: offline products order handling
When downloading an OFFLINE product in server mode:

- return HTTP Status 202 with new download link containing order status check information as query parameters
- don't retry to download the product again

Closes #871 

Also fixes missing `dataset` parameter after `BuildSearchResult` search-by-id in server mode

Patch:
diff --git a/eodag/plugins/download/http.py b/eodag/plugins/download/http.py
index 140f1e12..c1b03a36 100644
--- a/eodag/plugins/download/http.py
+++ b/eodag/plugins/download/http.py
@@ -130,7 +130,7 @@ class HTTPDownload(Download):
         product: EOProduct,
         auth: Optional[AuthBase] = None,
         **kwargs: Unpack[DownloadConf],
-    ) -> None:
+    ) -> Optional[Dict[str, Any]]:
         """Send product order request.
 
         It will be executed once before the download retry loop, if the product is OFFLINE
@@ -156,6 +156,8 @@ class HTTPDownload(Download):
         :type auth: Optional[AuthBase]
         :param kwargs: download additional kwargs
         :type kwargs: Union[str, bool, dict]
+        :returns: the returned json status response
+        :rtype: dict
         """
         product.properties["storageStatus"] = STAGING_STATUS
 
@@ -205,15 +207,20 @@ class HTTPDownload(Download):
                 )
                 self._check_auth_exception(e)
 
-            self.order_response_process(response, product)
+            return self.order_response_process(response, product)
+        return None
 
-    def order_response_process(self, response: Response, product: EOProduct) -> None:
+    def order_response_process(
+        self, response: Response, product: EOProduct
+    ) -> Optional[Dict[str, Any]]:
         """Process order response
 
         :param response: The order response
         :type response: :class:`~requests.Response`
         :param product: The orderd EO product
         :type product: :class:`~eodag.api.product._product.EOProduct`
+        :returns: the returned json status response
+        :rtype: dict
         """
         order_metadata_mapping = getattr(self.config, "order_on_response", {}).get(
             "metadata_mapping", {}
@@ -223,8 +230,11 @@ class HTTPDownload(Download):
             order_metadata_mapping_jsonpath = mtd_cfg_as_conversion_and_querypath(
                 order_metadata_mapping,
             )
+
+            json_response = response.json()
+
             properties_update = properties_from_json(
-                response.json(),
+                json_response,
                 order_metadata_mapping_jsonpath,
             )
             properties_update = {
@@ -235,8 +245,12 @@ class HTTPDownload(Download):
                 product.remote_location = product.location = product.properties[
                     "downloadLink"
                 ]
+                product.properties["storageStatus"] = ONLINE_STATUS
                 logger.debug(f"Product location updated to {product.location}")
 
+            return json_response
+        return None
+
     def orderDownloadStatus(
         self,
         product: EOProduct,
@@ -582,6 +596,19 @@ class HTTPDownload(Download):
             )
         return stream_size
 
+    def _check_product_filename(self, product: EOProduct) -> str:
+        filename = None
+        asset_content_disposition = self.stream.headers.get("content-disposition", None)
+        if asset_content_disposition:
+            filename = cast(
+                Optional[str],
+                parse_header(asset_content_disposition).get_param("filename", None),
+            )
+        if not filename:
+            # default filename extracted from path
+            filename = os.path.basename(self.stream.url)
+        return filename
+
     def _stream_download_dict(
         self,
         product: EOProduct,
@@ -823,7 +850,11 @@ class HTTPDownload(Download):
                         product.properties["storageStatus"] = "ORDERED"
                         self._process_exception(None, product, ordered_message)
                 stream_size = self._check_stream_size(product) or None
+                filename = self._check_product_filename(product) or None
                 product.headers = self.stream.headers
+                product.headers[
+                    "content-disposition"
+                ] = f"attachment; filename={filename}"
                 progress_callback.reset(total=stream_size)
                 for chunk in self.stream.iter_content(chunk_size=64 * 1024):
                     if chunk:
diff --git a/eodag/plugins/search/build_search_result.py b/eodag/plugins/search/build_search_result.py
index 396b5dd0..8e5f9e9e 100644
--- a/eodag/plugins/search/build_search_result.py
+++ b/eodag/plugins/search/build_search_result.py
@@ -175,8 +175,9 @@ class BuildPostSearchResult(PostJsonSearch):
 
         query_hash = hashlib.sha1(str(qs).encode("UTF-8")).hexdigest()
 
-        # update result with search args if not None (and not auth)
+        # update result with product_type_def_params and search args if not None (and not auth)
         kwargs.pop("auth", None)
+        result.update(self.product_type_def_params)
         result = dict(result, **{k: v for k, v in kwargs.items() if v is not None})
 
         # parse porperties
diff --git a/eodag/plugins/search/qssearch.py b/eodag/plugins/search/qssearch.py
index 7c97fc86..9e8c5a2d 100644
--- a/eodag/plugins/search/qssearch.py
+++ b/eodag/plugins/search/qssearch.py
@@ -1144,6 +1144,11 @@ class PostJsonSearch(QueryStringSearch):
         if _dc_qs is not None:
             qs = unquote_plus(unquote_plus(_dc_qs))
             qp = geojson.loads(qs)
+
+            # provider product type specific conf
+            self.product_type_def_params = self.get_product_type_def_params(
+                product_type, **kwargs
+            )
         else:
             keywords = {
                 k: v for k, v in kwargs.items() if k != "auth" and v is not None
diff --git a/eodag/rest/core.py b/eodag/rest/core.py
index 729e8f99..df5aa6c6 100644
--- a/eodag/rest/core.py
+++ b/eodag/rest/core.py
@@ -15,20 +15,28 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from __future__ import annotations
+
 import datetime
 import logging
 import os
 import re
-from typing import Any, Callable, Dict, List, Optional, cast
+from typing import TYPE_CHECKING, cast
+from unittest.mock import Mock
 
 import dateutil
-from fastapi import Request
-from fastapi.responses import StreamingResponse
+from fastapi.responses import ORJSONResponse, StreamingResponse
 from pydantic import ValidationError as pydanticValidationError
+from requests.models import Response as RequestsResponse
 
 import eodag
 from eodag import EOProduct
-from eodag.api.product.metadata_mapping import OSEO_METADATA_MAPPING
+from eodag.api.product.metadata_mapping import (
+    NOT_AVAILABLE,
+    ONLINE_STATUS,
+    OSEO_METADATA_MAPPING,
+    STAGING_STATUS,
+)
 from eodag.api.search_result import SearchResult
 from eodag.config import load_stac_config
 from eodag.plugins.crunch.filter_latest_intersect import FilterLatestIntersect
@@ -37,7 +45,6 @@ from eodag.plugins.crunch.filter_overlap import FilterOverlap
 from eodag.rest.stac import StacCatalog, StacCollection, StacCommon, StacItem
 from eodag.rest.types.eodag_search import EODAGSearch
 from eodag.rest.types.stac_queryables import StacQueryableProperty, StacQueryables
-from eodag.rest.types.stac_search import SearchPostRequest
 from eodag.rest.utils import (
     Cruncher,
     file_to_stream,
@@ -51,6 +58,7 @@ from eodag.utils import (
     _deprecated,
     deepcopy,
     dict_items_recursive_apply,
+    urlencode,
 )
 from eodag.utils.exceptions import (
     MisconfiguredError,
@@ -59,6 +67,16 @@ from eodag.utils.exceptions import (
     ValidationError,
 )
 
+if TYPE_CHECKING:
+    from typing import Any, Callable, Dict, List, Optional, Union
+
+    from fastapi import Request
+    from requests.auth import AuthBase
+    from starlette.responses import Response
+
+    from eodag.rest.types.stac_search import SearchPostRequest
+
+
 eodag_api = eodag.EODataAccessGateway()
 
 logger = logging.getLogger("eodag.rest.core")
@@ -212,12 +230,13 @@ def search_stac_items(
 
 
 def download_stac_item(
+    request: Request,
     catalogs: List[str],
     item_id: str,
     provider: Optional[str] = None,
     asset: Optional[str] = None,
     **kwargs: Any,
-) -> StreamingResponse:
+) -> Response:
     """Download item
 
     :param catalogs: Catalogs list (only first is used as product_type)
@@ -229,7 +248,7 @@ def download_stac_item(
     :param kwargs: additional download parameters
     :type kwargs: Any
     :returns: a stream of the downloaded data (zip file)
-    :rtype: StreamingResponse
+    :rtype: Response
     """
     product_type = catalogs[0]
 
@@ -243,12 +262,19 @@ def download_stac_item(
             f"Could not find {item_id} item in {product_type} collection"
             + (f" for provider {provider}" if provider else "")
         )
+    auth = product.downloader_auth.authenticate()
 
     try:
+        _order_and_update(product, auth, **kwargs)
+
         download_stream = cast(
             StreamResponse,
             product.downloader._stream_download_dict(
-                product, auth=product.downloader_auth.authenticate(), asset=asset
+                product,
+                auth=auth,
+                asset=asset,
+                wait=-1,
+                timeout=-1,
             ),
         )
     except NotImplementedError:
@@ -259,6 +285,19 @@ def download_stac_item(
         download_stream = file_to_stream(
             eodag_api.download(product, extract=False, asset=asset)
         )
+    except NotAvailableError:
+        if product.properties.get("storageStatus") != ONLINE_STATUS:
+            download_link = f"{request.state.url}?{urlencode(dict(kwargs, **{'provider': provider}), doseq=True)}"
+            return ORJSONResponse(
+                status_code=202,
+                headers={"Location": download_link},
+                content={
+                    "description": "Product is not available yet, please try again using given updated location",
+                    "location": download_link,
+                },
+            )
+        else:
+            raise
 
     return StreamingResponse(
         content=download_stream.content,
@@ -267,6 +306,40 @@ def download_stac_item(
     )
 
 
+def _order_and_update(
+    product: EOProduct, auth: Union[AuthBase, Dict[str, str]], **kwargs: Any
+) -> None:
+    """Order product if needed and update given kwargs with order-status-dict"""
+    if product.properties.get("storageStatus") != ONLINE_STATUS and hasattr(
+        product.downloader, "order_response_process"
+    ):
+        # update product (including orderStatusLink) if product was previously ordered
+        logger.debug("Use given download query arguments to parse order link")
+        response = Mock(spec=RequestsResponse)
+        response.status_code = 200
+        response.json.return_value = kwargs
+        product.downloader.order_response_process(response, product)
+
+    if (
+        product.properties.get("storageStatus") != ONLINE_STATUS
+        and NOT_AVAILABLE in product.properties.get("orderStatusLink", "")
+        and hasattr(product.downloader, "orderDownload")
+    ):
+        # first order
+        logger.debug("Order product")
+        order_status_dict = product.downloader.orderDownload(product=product, auth=auth)
+        kwargs.update(order_status_dict or {})
+
+    if product.properties.get("storageStatus") == STAGING_STATUS and hasattr(
+        product.downloader, "orderDownloadStatus"
+    ):
+        # check order status if needed
+        logger.debug("Checking product order status")
+        product.downloader.orderDownloadStatus(product=product, auth=auth)
+        if product.properties.get("storageStatus") != ONLINE_STATUS:
+            raise NotAvailableError("Product is not available yet")
+
+
 def get_detailled_collections_list(
     provider: Optional[str] = None, fetch_providers: bool = True
 ) -> List[Dict[str, Any]]:
diff --git a/eodag/rest/server.py b/eodag/rest/server.py
index 3aedb6a9..21139461 100755
--- a/eodag/rest/server.py
+++ b/eodag/rest/server.py
@@ -39,7 +39,7 @@ from fastapi import FastAPI, HTTPException, Request
 from fastapi.encoders import jsonable_encoder
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.openapi.utils import get_openapi
-from fastapi.responses import ORJSONResponse, StreamingResponse
+from fastapi.responses import ORJSONResponse
 from pydantic import ValidationError as pydanticValidationError
 from pygeofilter.backends.cql2_json import to_cql2
 from pygeofilter.parsers.cql2_text import parse as parse_cql2_text
@@ -80,6 +80,8 @@ if TYPE_CHECKING:
     from fastapi.types import DecoratedCallable
     from requests import Response
 
+from starlette.responses import Response as StarletteResponse
+
 logger = logging.getLogger("eodag.rest.server")
 ERRORS_WITH_500_STATUS_CODE = {
     "MisconfiguredError",
@@ -385,7 +387,7 @@ def stac_extension_oseo(request: Request) -> Any:
 )
 def stac_collections_item_download(
     collection_id: str, item_id: str, request: Request
-) -> StreamingResponse:
+) -> StarletteResponse:
     """STAC collection item download"""
     logger.debug("URL: %s", request.url)
 
@@ -393,7 +395,11 @@ def stac_collections_item_download(
     provider = arguments.pop("provider", None)
 
     return download_stac_item(
-        catalogs=[collection_id], item_id=item_id, provider=provider, **arguments
+        request=request,
+        catalogs=[collection_id],
+        item_id=item_id,
+        provider=provider,
+        **arguments,
     )
 
 
@@ -412,6 +418,7 @@ def stac_collections_item_download_asset(
     provider = arguments.pop("provider", None)
 
     return download_stac_item(
+        request=request,
         catalogs=[collection_id],
         item_id=item_id,
         provider=provider,
@@ -575,7 +582,7 @@ def collections(request: Request) -> Any:
 )
 def stac_catalogs_item_download(
     catalogs: str, item_id: str, request: Request
-) -> StreamingResponse:
+) -> StarletteResponse:
     """STAC Catalog item download"""
     logger.debug("URL: %s", request.url)
 
@@ -585,7 +592,11 @@ def stac_catalogs_item_download(
     list_catalog = catalogs.strip("/").split("/")
 
     return download_stac_item(
-        catalogs=list_catalog, item_id=item_id, provider=provider, **arguments
+        request=request,
+        catalogs=list_catalog,
+        item_id=item_id,
+        provider=provider,
+        **arguments,
     )
 
 
@@ -606,6 +617,7 @@ def stac_catalogs_item_download_asset(
     list_catalog = catalogs.strip("/").split("/")
 
     return download_stac_item(
+        request=request,
         catalogs=list_catalog,
         item_id=item_id,
         provider=provider,
diff --git a/eodag/utils/__init__.py b/eodag/utils/__init__.py
index 6471d0db..b46a838a 100644
--- a/eodag/utils/__init__.py
+++ b/eodag/utils/__init__.py
@@ -1433,6 +1433,7 @@ class StreamResponse:
     content: Iterator[bytes]
     headers: Optional[Mapping[str, str]] = None
     media_type: Optional[str] = None
+    status_code: Optional[int] = None
 
 
 def guess_file_type(file: str) -> Optional[str]:


Test patch:
diff --git a/tests/__init__.py b/tests/__init__.py
index 2bd14e8a..59ce5692 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -335,6 +335,7 @@ class EODagTestCase(unittest.TestCase):
                     response.__zip_buffer = io.BytesIO(fh.read())
                 cl = response.__zip_buffer.getbuffer().nbytes
                 response.headers = {"content-length": cl}
+                response.url = "http://foo.bar"
 
             def __enter__(response):
                 return response
diff --git a/tests/context.py b/tests/context.py
index 2e6aeec7..f697bb45 100644
--- a/tests/context.py
+++ b/tests/context.py
@@ -33,6 +33,7 @@ from eodag.api.product.metadata_mapping import (
     format_metadata,
     OFFLINE_STATUS,
     ONLINE_STATUS,
+    STAGING_STATUS,
     properties_from_json,
     NOT_AVAILABLE,
 )
diff --git a/tests/units/test_download_plugins.py b/tests/units/test_download_plugins.py
index 832c63f7..3f2f79ef 100644
--- a/tests/units/test_download_plugins.py
+++ b/tests/units/test_download_plugins.py
@@ -172,6 +172,7 @@ class TestDownloadPluginHttp(BaseDownloadPluginTest):
                     self.__zip_buffer = io.BytesIO(fh.read())
                 cl = self.__zip_buffer.getbuffer().nbytes
                 self.headers = {"content-length": cl}
+                self.url = "http://foo.bar"
 
             def __enter__(self):
                 return self
@@ -1008,12 +1009,12 @@ class TestDownloadPluginHttp(BaseDownloadPluginTest):
         # empty product download directory should have been removed
         self.assertFalse(Path(os.path.join(self.output_dir, "dummy_product")).exists())
 
-    def test_plugins_download_http_order_download_cds(
+    def test_plugins_download_http_order_download_cop_ads(
         self,
     ):
         """HTTPDownload.download must order the product if needed"""
 
-        self.product.provider = "cop_cds"
+        self.product.provider = "cop_ads"
         self.product.product_type = "CAMS_EAC4"
         product_dataset = "cams-global-reanalysis-eac4"
 
@@ -1075,6 +1076,7 @@ class TestDownloadPluginHttp(BaseDownloadPluginTest):
 
             # expected values
             expected_remote_location = "http://somewhere/download/dummy_request_id"
+            expected_order_status_link = f"{endpoint}/tasks/dummy_request_id"
             expected_path = os.path.join(
                 output_data_path, self.product.properties["title"]
             )
@@ -1085,6 +1087,12 @@ class TestDownloadPluginHttp(BaseDownloadPluginTest):
                 timeout=0.2 / 60,
             )
             self.assertEqual(self.product.remote_location, expected_remote_location)
+            self.assertEqual(
+                self.product.properties["downloadLink"], expected_remote_location
+            )
+            self.assertEqual(
+                self.product.properties["orderStatusLink"], expected_order_status_link
+            )
             self.assertEqual(path, expected_path)
             self.assertEqual(self.product.location, path_to_uri(expected_path))
 
diff --git a/tests/units/test_http_server.py b/tests/units/test_http_server.py
index 9571d73f..5d72bed1 100644
--- a/tests/units/test_http_server.py
+++ b/tests/units/test_http_server.py
@@ -25,7 +25,7 @@ import unittest
 from pathlib import Path
 from tempfile import TemporaryDirectory
 from typing import Any, Dict, List, Optional, Union
-from unittest.mock import Mock
+from unittest.mock import MagicMock, Mock
 
 import geojson
 import httpx
@@ -36,11 +36,15 @@ from eodag.config import PluginConfig
 from eodag.plugins.authentication.base import Authentication
 from eodag.plugins.download.base import Download
 from eodag.utils import USER_AGENT, MockResponse, StreamResponse
-from eodag.utils.exceptions import TimeOutError
+from eodag.utils.exceptions import NotAvailableError, TimeOutError
 from tests import mock
 from tests.context import (
     DEFAULT_ITEMS_PER_PAGE,
     HTTP_REQ_TIMEOUT,
+    NOT_AVAILABLE,
+    OFFLINE_STATUS,
+    ONLINE_STATUS,
+    STAGING_STATUS,
     TEST_RESOURCES_PATH,
     AuthenticationError,
     SearchResult,
@@ -436,6 +440,13 @@ class RequestTestCase(unittest.TestCase):
         self.assertIn("description", response_content)
         self.assertIn("NotAvailableError", response_content["description"])
 
+    def _request_accepted(self, url: str):
+        response = self.app.get(url, follow_redirects=True)
+        response_content = json.loads(response.content.decode("utf-8"))
+        self.assertEqual(202, response.status_code)
+        self.assertIn("description", response_content)
+        self.assertIn("location", response_content)
+
     def test_request_params(self):
         self._request_not_valid(f"search?collections={self.tested_product_type}&bbox=1")
         self._request_not_valid(
@@ -1047,6 +1058,10 @@ class RequestTestCase(unittest.TestCase):
         response_filename = header_content_disposition.get_param("filename", None)
         self.assertEqual(response_filename, expected_file)
 
+    @mock.patch(
+        "eodag.plugins.authentication.base.Authentication.authenticate",
+        autospec=True,
+    )
     @mock.patch(
         "eodag.plugins.download.base.Download._stream_download_dict",
         autospec=True,
@@ -1056,7 +1071,7 @@ class RequestTestCase(unittest.TestCase):
         autospec=True,
     )
     def test_download_item_from_collection_no_stream(
-        self, mock_download: Mock, mock_stream_download: Mock
+        self, mock_download: Mock, mock_stream_download: Mock, mock_auth: Mock
     ):
         """Download through eodag server catalog should return a valid response"""
         # download should be performed locally then deleted if streaming is not available
@@ -1073,6 +1088,71 @@ class RequestTestCase(unittest.TestCase):
             expected_file
         ), f"File {expected_file} should have been deleted"
 
+    @mock.patch(
+        "eodag.rest.core.eodag_api.search",
+        autospec=True,
+    )
+    def test_download_offline_item_from_catalog(self, mock_search):
+        """Download an offline item through eodag server catalog should return a
+        response with HTTP Status 202"""
+        # mock_search_result returns 2 search results, only keep one
+        two_results = self.mock_search_result()
+        product = two_results[0][0]
+        mock_search.return_value = (
+            [
+                product,
+            ],
+            1,
+        )
+        product.downloader_auth = MagicMock()
+        product.downloader.orderDownload = MagicMock()
+        product.downloader.orderDownloadStatus = MagicMock()
+        product.downloader.order_response_process = MagicMock()
+        product.downloader._stream_download_dict = MagicMock(
+            side_effect=NotAvailableError("Product offline. Try again later.")
+        )
+        product.properties["orderStatusLink"] = f"{NOT_AVAILABLE}?foo=bar"
+
+        # ONLINE product with error
+        product.properties["storageStatus"] = ONLINE_STATUS
+        # status 404 and no order try
+        self._request_not_found(
+            f"catalogs/{self.tested_product_type}/items/foo/download"
+        )
+        product.downloader.orderDownload.assert_not_called()
+        product.downloader.orderDownloadStatus.assert_not_called()
+        product.downloader.order_response_process.assert_not_called()
+        product.downloader._stream_download_dict.assert_called_once()
+        product.downloader._stream_download_dict.reset_mock()
+
+        # OFFLINE product with error
+        product.properties["storageStatus"] = OFFLINE_STATUS
+        # status 202 and order once and no status check
+        self._request_accepted(
+            f"catalogs/{self.tested_product_type}/items/foo/download"
+        )
+        product.downloader.orderDownload.assert_called_once()
+        product.downloader.orderDownload.reset_mock()
+        product.downloader.orderDownloadStatus.assert_not_called()
+        product.downloader.order_response_process.assert_called()
+        product.downloader.order_response_process.reset_mock()
+        product.downloader._stream_download_dict.assert_called_once()
+        product.downloader._stream_download_dict.reset_mock()
+
+        # STAGING product and available orderStatusLink
+        product.properties["storageStatus"] = STAGING_STATUS
+        product.properties["orderStatusLink"] = "http://somewhere?foo=bar"
+        # status 202 and no order but status checked and no download try
+        self._request_accepted(
+            f"catalogs/{self.tested_product_type}/items/foo/download"
+        )
+        product.downloader.orderDownload.assert_not_called()
+        product.downloader.orderDownloadStatus.assert_called_once()
+        product.downloader.orderDownloadStatus.reset_mock()
+        product.downloader.order_response_process.assert_called()
+        product.downloader.order_response_process.reset_mock()
+        product.downloader._stream_download_dict.assert_not_called()
+
     def test_conformance(self):
         """Request to /conformance should return a valid response"""
         self._request_valid("conformance", check_links=False)