Repository: CS-SI/eodag
Pull request #1005 (2024-01-26 17:25:15)

refactor: queryables type to Annotated
Following https://github.com/CS-SI/eodag/pull/974 (no release between these 2 Pull Requests), 

changes `list_queryables()` output from 
`Dict[str, Tuple[Annotated[Any, FieldInfo], Any]]`
to
`Dict[str, Annotated[Any, FieldInfo]]`.

Previous format returned *field default value* in the the 2nd argument of the tuple.
This is not needed any more as this value can be included in `FieldInfo`.

Also, `pydantic.create_model` can be still be used with `list_queryables()` output:
- before:
```py
>>> from pydantic import create_model
>>> from eodag import EODataAccessGateway
>>> dag = EODataAccessGateway()
>>> queryables = dag.list_queryables(product_type="S2_MSI_L1C", provider="peps")
>>> queryables
{'productType': (typing.Annotated[str, FieldInfo(annotation=NoneType, required=True)], None), ...}
>>> create_model("my_model", **queryables)
<class 'pydantic.main.my_model'>
```
- now:
```py
>>> from pydantic import create_model
>>> from typing_extensions import get_args
>>> from eodag import EODataAccessGateway
>>> dag = EODataAccessGateway()
>>> queryables = dag.list_queryables(product_type="S2_MSI_L1C", provider="peps")
>>> queryables
{'productType': typing.Annotated[str, FieldInfo(annotation=NoneType, required=True)], ...}
>>> create_model("my_model", **{k: get_args(v) for k, v in queryables.items()})
<class 'pydantic.main.my_model'>
```

github-actions[bot]: ## Test Results
    4 files  ±0      4 suites  ±0   4m 31s :stopwatch: +2s
  455 tests ±0    452 :white_check_mark: ±0   3 :zzz: ±0  0 :x: ±0 
1 820 runs  ±0  1 740 :white_check_mark: ±0  80 :zzz: ±0  0 :x: ±0 

Results for commit b5e8e18c. ± Comparison against base commit 932eb17c.

<details>
  <summary>This pull request <b>removes</b> 1 and <b>adds</b> 1 tests. <i>Note that renamed tests count towards both.</i></summary>

```
eodag.types.__init__ ‑ eodag.types.model_fields_to_annotated_tuple
```

```
eodag.types.__init__ ‑ eodag.types.model_fields_to_annotated
```
</details>

[test-results]:data:application/gzip;base64,H4sIAMLrs2UC/03MSw7CIBSF4a00jB2UZ6mbMXi5JMS2GB4j497FKrTD/zvJeRHnF0zkOojLQFLxuYct0WQftppsohXqlPdRyla3VAB2Ygc9/LMS7+CMXyqMHTDGEP8Sy/b9pJq1ap90Egf9PnWH0+fe50sI6+pzDXKXqJFqkGqC0VLLBVKYGc5KcKEURxTgwHDy/gDIOJBZCAEAAA==

github-actions[bot]: <strong>Code Coverage (Ubuntu)</strong>

| File | Coverage |   |
| - | :-: | :-: |
| **All files** | `82%` | :white_check_mark: |

_Minimum allowed coverage is `70%`_

<p align="right">Generated by :monkey: cobertura-action against b5e8e18c567c0d1d34e1c92e96434663ee4cfca3 </p>
github-actions[bot]: <strong>Code Coverage (Windows)</strong>

| File | Coverage |   |
| - | :-: | :-: |
| **All files** | `76%` | :white_check_mark: |

_Minimum allowed coverage is `70%`_

<p align="right">Generated by :monkey: cobertura-action against b5e8e18c567c0d1d34e1c92e96434663ee4cfca3 </p>

Patch:
diff --git a/eodag/api/core.py b/eodag/api/core.py
index c7bb0c92..80d06a08 100644
--- a/eodag/api/core.py
+++ b/eodag/api/core.py
@@ -22,12 +22,24 @@ import os
 import re
 import shutil
 from operator import itemgetter
-from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Set, Tuple, Union
+from typing import (
+    TYPE_CHECKING,
+    AbstractSet,
+    Any,
+    Dict,
+    Iterator,
+    List,
+    Optional,
+    Set,
+    Tuple,
+    Union,
+)
 
 import geojson
 import pkg_resources
 import yaml.parser
 from pkg_resources import resource_filename
+from pydantic.fields import FieldInfo
 from whoosh import analysis, fields
 from whoosh.fields import Schema
 from whoosh.index import create_in, exists_in, open_dir
@@ -51,7 +63,7 @@ from eodag.config import (
 )
 from eodag.plugins.manager import PluginManager
 from eodag.plugins.search.build_search_result import BuildPostSearchResult
-from eodag.types import model_fields_to_annotated_tuple
+from eodag.types import model_fields_to_annotated
 from eodag.types.queryables import CommonQueryables, Queryables
 from eodag.utils import (
     DEFAULT_DOWNLOAD_TIMEOUT,
@@ -83,7 +95,6 @@ from eodag.utils.exceptions import (
 from eodag.utils.stac_reader import fetch_stac_items
 
 if TYPE_CHECKING:
-    from pydantic.fields import FieldInfo
     from shapely.geometry.base import BaseGeometry
     from whoosh.index import Index
 
@@ -2092,7 +2103,7 @@ class EODataAccessGateway:
         self,
         provider: Optional[str] = None,
         product_type: Optional[str] = None,
-    ) -> Dict[str, Tuple[Annotated[Any, FieldInfo], Any]]:
+    ) -> Dict[str, Annotated[Any, FieldInfo]]:
         """Fetch the queryable properties for a given product type and/or provider.
 
         :param provider: (optional) The provider.
@@ -2100,9 +2111,8 @@ class EODataAccessGateway:
         :param product_type: (optional) The EODAG product type.
         :type product_type: str
         :returns: A dict containing the EODAG queryable properties, associating
-                  parameters to a tuple containing their annotaded type and default
-                  value
-        :rtype: Dict[str, Tuple[Annotated[Any, FieldInfo], Any]]
+                  parameters to their annotaded type
+        :rtype: Dict[str, Annotated[Any, FieldInfo]]
         """
         # unknown product type
         if product_type is not None and product_type not in self.list_product_types(
@@ -2112,11 +2122,11 @@ class EODataAccessGateway:
 
         # dictionary of the queryable properties of the providers supporting the given product type
         providers_available_queryables: Dict[
-            str, Dict[str, Tuple[Annotated[Any, FieldInfo], Any]]
+            str, Dict[str, Annotated[Any, FieldInfo]]
         ] = dict()
 
         if provider is None and product_type is None:
-            return model_fields_to_annotated_tuple(CommonQueryables.model_fields)
+            return model_fields_to_annotated(CommonQueryables.model_fields)
         elif provider is None:
             for plugin in self._plugins_manager.get_search_plugins(
                 product_type, provider
@@ -2126,7 +2136,7 @@ class EODataAccessGateway:
                 )
 
             # return providers queryables intersection
-            queryables_keys = {}
+            queryables_keys: AbstractSet[str] = set()
             for queryables in providers_available_queryables.values():
                 queryables_keys = (
                     queryables_keys & queryables.keys()
@@ -2139,7 +2149,7 @@ class EODataAccessGateway:
                 if k in queryables_keys
             }
 
-        all_queryables = model_fields_to_annotated_tuple(Queryables.model_fields)
+        all_queryables = model_fields_to_annotated(Queryables.model_fields)
 
         try:
             plugin = next(
@@ -2147,7 +2157,7 @@ class EODataAccessGateway:
             )
         except StopIteration:
             # return default queryables if no plugin is found
-            return model_fields_to_annotated_tuple(CommonQueryables.model_fields)
+            return model_fields_to_annotated(CommonQueryables.model_fields)
 
         providers_available_queryables[plugin.provider] = dict()
 
@@ -2179,10 +2189,12 @@ class EODataAccessGateway:
                 del metadata_mapping[param]
 
         for key, value in all_queryables.items():
-            annotated_args = get_args(value[0])
+            annotated_args = get_args(value)
             if len(annotated_args) < 1:
                 continue
             field_info = annotated_args[1]
+            if not isinstance(field_info, FieldInfo):
+                continue
             if field_info.is_required() or (
                 (field_info.alias or key) in metadata_mapping
             ):
@@ -2193,7 +2205,7 @@ class EODataAccessGateway:
         provider_queryables.update(providers_available_queryables[provider])
         # always keep at least CommonQueryables
         provider_queryables.update(
-            model_fields_to_annotated_tuple(CommonQueryables.model_fields)
+            model_fields_to_annotated(CommonQueryables.model_fields)
         )
 
         return provider_queryables
diff --git a/eodag/plugins/apis/base.py b/eodag/plugins/apis/base.py
index ab7f1497..a5a3b1dc 100644
--- a/eodag/plugins/apis/base.py
+++ b/eodag/plugins/apis/base.py
@@ -97,7 +97,7 @@ class Api(PluginTopic):
 
     def discover_queryables(
         self, product_type: Optional[str] = None
-    ) -> Optional[Dict[str, Tuple[Annotated[Any, FieldInfo], Any]]]:
+    ) -> Optional[Dict[str, Annotated[Any, FieldInfo]]]:
         """Fetch queryables list from provider using `discover_queryables` conf"""
         return None
 
diff --git a/eodag/plugins/search/base.py b/eodag/plugins/search/base.py
index fed7ca11..5b087fa0 100644
--- a/eodag/plugins/search/base.py
+++ b/eodag/plugins/search/base.py
@@ -92,7 +92,7 @@ class Search(PluginTopic):
 
     def discover_queryables(
         self, product_type: Optional[str] = None
-    ) -> Optional[Dict[str, Tuple[Annotated[Any, FieldInfo], Any]]]:
+    ) -> Optional[Dict[str, Annotated[Any, FieldInfo]]]:
         """Fetch queryables list from provider using `discover_queryables` conf"""
         return None
 
diff --git a/eodag/plugins/search/qssearch.py b/eodag/plugins/search/qssearch.py
index a4034626..002087f6 100644
--- a/eodag/plugins/search/qssearch.py
+++ b/eodag/plugins/search/qssearch.py
@@ -43,7 +43,7 @@ from eodag.api.product.metadata_mapping import (
     properties_from_xml,
 )
 from eodag.plugins.search.base import Search
-from eodag.types import json_field_definition_to_python, model_fields_to_annotated_tuple
+from eodag.types import json_field_definition_to_python, model_fields_to_annotated
 from eodag.utils import (
     DEFAULT_ITEMS_PER_PAGE,
     DEFAULT_PAGE,
@@ -55,6 +55,7 @@ from eodag.utils import (
     deepcopy,
     dict_items_recursive_apply,
     format_dict_items,
+    get_args,
     quote,
     string_to_jsonpath,
     update_nested_dict,
@@ -1282,7 +1283,7 @@ class StacSearch(PostJsonSearch):
 
     def discover_queryables(
         self, product_type: Optional[str] = None
-    ) -> Optional[Dict[str, Tuple[Annotated[Any, FieldInfo], Any]]]:
+    ) -> Optional[Dict[str, Annotated[Any, FieldInfo]]]:
         """Fetch queryables list from provider using `discover_queryables` conf
 
         :param product_type: (optional) product type
@@ -1290,12 +1291,12 @@ class StacSearch(PostJsonSearch):
         :returns: fetched queryable parameters dict
         :rtype: dict
         """
-        provider_product_type = self.config.products.get(product_type, {}).get(
-            "productType", product_type
+        provider_product_type = (
+            self.config.products.get(product_type, {}).get("productType", product_type)
+            if product_type
+            else None
         )
 
-        python_queryables: Dict[str, FieldInfo] = dict()
-
         try:
             unparsed_fetch_url = (
                 self.config.discover_queryables["product_type_fetch_url"]
@@ -1341,7 +1342,7 @@ class StacSearch(PostJsonSearch):
                 return None
 
             # convert json results to pydantic model fields
-            field_definitions = dict()
+            field_definitions: Dict[str, Any] = dict()
             for json_param, json_mtd in json_queryables.items():
                 param = (
                     get_queryable_from_provider(
@@ -1349,8 +1350,9 @@ class StacSearch(PostJsonSearch):
                     )
                     or json_param
                 )
-                field_definitions[param] = json_field_definition_to_python(json_mtd)
+                annotated_def = json_field_definition_to_python(json_mtd)
+                field_definitions[param] = get_args(annotated_def)
 
             python_queryables = create_model("m", **field_definitions).model_fields
 
-        return model_fields_to_annotated_tuple(python_queryables)
+        return model_fields_to_annotated(python_queryables)
diff --git a/eodag/rest/types/stac_queryables.py b/eodag/rest/types/stac_queryables.py
index e395ea97..c4a1fa62 100644
--- a/eodag/rest/types/stac_queryables.py
+++ b/eodag/rest/types/stac_queryables.py
@@ -17,7 +17,7 @@
 # limitations under the License.
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
 
 from pydantic import BaseModel, Field
 
@@ -52,7 +52,7 @@ class StacQueryableProperty(BaseModel):
 
     @classmethod
     def from_python_field_definition(
-        cls, id: str, python_field_definition: Tuple[Annotated[Any, FieldInfo], Any]
+        cls, id: str, python_field_definition: Annotated[Any, FieldInfo]
     ) -> StacQueryableProperty:
         """Build Model from python_field_definition"""
         def_dict = python_field_definition_to_json(python_field_definition)
diff --git a/eodag/types/__init__.py b/eodag/types/__init__.py
index 0e7b1b5a..a29d40a2 100644
--- a/eodag/types/__init__.py
+++ b/eodag/types/__init__.py
@@ -18,7 +18,7 @@
 """EODAG types"""
 from __future__ import annotations
 
-from typing import Any, Dict, List, Literal, Optional, Tuple, Union
+from typing import Any, Dict, List, Literal, Optional, Union
 
 from pydantic import Field
 from pydantic.fields import FieldInfo
@@ -34,6 +34,7 @@ JSON_TYPES_MAPPING: Dict[str, type] = {
     "number": float,
     "string": str,
     "array": list,
+    "object": dict,
     "null": type(None),
 }
 
@@ -48,11 +49,11 @@ def json_type_to_python(json_type: Union[str, List[str]]) -> type:
     :returns: the python type
     """
     if isinstance(json_type, list) and len(json_type) > 1:
-        return Union[tuple(JSON_TYPES_MAPPING.get(jt, Any) for jt in json_type)]  # type: ignore
+        return Union[tuple(JSON_TYPES_MAPPING.get(jt, type(None)) for jt in json_type)]  # type: ignore
     elif isinstance(json_type, str):
-        return JSON_TYPES_MAPPING.get(json_type, Any)
+        return JSON_TYPES_MAPPING.get(json_type, type(None))
     else:
-        return Any
+        return type(None)
 
 
 def python_type_to_json(python_type: type) -> Optional[Union[str, List[str]]]:
@@ -86,8 +87,8 @@ def python_type_to_json(python_type: type) -> Optional[Union[str, List[str]]]:
 
 
 def json_field_definition_to_python(
-    json_field_definition: Dict[str, Any], default_value: Optional[Any] = None
-) -> Annotated:
+    json_field_definition: Dict[str, Any]
+) -> Annotated[Any, FieldInfo]:
     """Get python field definition from json object
 
     >>> result = json_field_definition_to_python(
@@ -97,7 +98,7 @@ def json_field_definition_to_python(
     ...     }
     ... )
     >>> str(result).replace('_extensions', '') # python3.8 compatibility
-    "(typing.Annotated[bool, FieldInfo(annotation=NoneType, required=False, title='Foo parameter')], None)"
+    "typing.Annotated[bool, FieldInfo(annotation=NoneType, required=False, title='Foo parameter')]"
 
     :param json_field_definition: the json field definition
     :returns: the python field definition
@@ -118,46 +119,35 @@ def json_field_definition_to_python(
     if "$ref" in json_field_definition:
         field_type_kwargs["json_schema_extra"] = {"$ref": json_field_definition["$ref"]}
 
-    return (Annotated[python_type, Field(None, **field_type_kwargs)], default_value)
+    return Annotated[python_type, Field(None, **field_type_kwargs)]
 
 
 def python_field_definition_to_json(
-    python_field_definition: Tuple[Annotated[Any, FieldInfo], Any]
+    python_field_definition: Annotated[Any, FieldInfo]
 ) -> Dict[str, Any]:
     """Get json field definition from python `typing.Annotated`
 
     >>> from pydantic import Field
     >>> from eodag.utils import Annotated
     >>> python_field_definition_to_json(
-    ...     (Annotated[
+    ...     Annotated[
     ...         Optional[str],
     ...         Field(None, description='Foo bar', json_schema_extra={'$ref': '/path/to/schema'})
-    ...     ], None)
+    ...     ]
     ... )
     {'type': ['string', 'null'], 'description': 'Foo bar', '$ref': '/path/to/schema'}
 
-    :param python_field_definition: the python field definition (tuple of annotated type, default value)
-    :returns: the python field definition
+    :param python_field_annotated: the python field annotated type
+    :returns: the json field definition
     """
-    if (
-        not isinstance(python_field_definition, tuple)
-        or len(python_field_definition) != 2
-    ):
-        raise ValidationError(
-            "%s must be an instance of Tuple[Annotated[Any, FieldInfo], Any]"
-            % python_field_definition
-        )
-
-    python_field_annotated = python_field_definition[0]
-
-    if get_origin(python_field_annotated) is not Annotated:
+    if get_origin(python_field_definition) is not Annotated:
         raise ValidationError(
-            "%s must be an instance of Annotated" % python_field_annotated
+            "%s must be an instance of Annotated" % python_field_definition
         )
 
     json_field_definition = dict()
 
-    python_field_args = get_args(python_field_annotated)
+    python_field_args = get_args(python_field_definition)
 
     # enum & type
     if get_origin(python_field_args[0]) is Literal:
@@ -195,19 +185,16 @@ def python_field_definition_to_json(
     return json_field_definition
 
 
-def model_fields_to_annotated_tuple(
+def model_fields_to_annotated(
     model_fields: Dict[str, FieldInfo]
-) -> Dict[str, Tuple[Annotated[Any, FieldInfo], Any]]:
-    """Convert BaseModel.model_fields from FieldInfo to Annotated tuple usable as create_model argument
+) -> Dict[str, Annotated[Any, FieldInfo]]:
+    """Convert BaseModel.model_fields from FieldInfo to Annotated
 
     >>> from pydantic import create_model
     >>> some_model = create_model("some_model", foo=(str, None))
-    >>> fields_definitions = model_fields_to_annotated_tuple(some_model.model_fields)
+    >>> fields_definitions = model_fields_to_annotated(some_model.model_fields)
     >>> str(fields_definitions).replace('_extensions', '') # python3.8 compatibility
-    "{'foo': (typing.Annotated[str, FieldInfo(annotation=NoneType, required=False)], None)}"
-    >>> another_model = create_model("another_model", **fields_definitions)
-    >>> another_model(foo="abc")
-    another_model(foo='abc')
+    "{'foo': typing.Annotated[str, FieldInfo(annotation=NoneType, required=False)]}"
 
     :param model_fields: BaseModel.model_fields to convert
     :returns: Annotated tuple usable as create_model argument
@@ -217,5 +204,5 @@ def model_fields_to_annotated_tuple(
         field_type = field_info.annotation or type(None)
         new_field_info = copy_deepcopy(field_info)
         new_field_info.annotation = None
-        annotated_model_fields[param] = (Annotated[field_type, new_field_info], None)
+        annotated_model_fields[param] = Annotated[field_type, new_field_info]
     return annotated_model_fields


Test patch:
diff --git a/tests/context.py b/tests/context.py
index 41573767..72b07403 100644
--- a/tests/context.py
+++ b/tests/context.py
@@ -65,7 +65,7 @@ from eodag.plugins.download.http import HTTPDownload
 from eodag.plugins.manager import PluginManager
 from eodag.plugins.search.base import Search
 from eodag.rest.stac import DEFAULT_MISSION_START_DATE
-from eodag.types import model_fields_to_annotated_tuple
+from eodag.types import model_fields_to_annotated
 from eodag.types.queryables import CommonQueryables, Queryables
 from eodag.utils import (
     DEFAULT_STREAM_REQUESTS_TIMEOUT,
diff --git a/tests/units/test_core.py b/tests/units/test_core.py
index f1da82c3..f5f118c4 100644
--- a/tests/units/test_core.py
+++ b/tests/units/test_core.py
@@ -50,7 +50,7 @@ from tests.context import (
     get_geometry_from_various,
     load_default_config,
     makedirs,
-    model_fields_to_annotated_tuple,
+    model_fields_to_annotated,
 )
 from tests.utils import mock, write_eodag_conf_with_fake_credentials
 
@@ -1090,16 +1090,14 @@ class TestCore(TestCoreBase):
             self.dag.list_queryables(product_type="not_existing_product_type")
 
         queryables_none_none = self.dag.list_queryables()
-        expected_result = model_fields_to_annotated_tuple(CommonQueryables.model_fields)
+        expected_result = model_fields_to_annotated(CommonQueryables.model_fields)
         self.assertEqual(len(queryables_none_none), len(expected_result))
         for key, queryable in queryables_none_none.items():
             # compare obj.__repr__
             self.assertEqual(str(expected_result[key]), str(queryable))
 
         queryables_peps_none = self.dag.list_queryables(provider="peps")
-        expected_longer_result = model_fields_to_annotated_tuple(
-            Queryables.model_fields
-        )
+        expected_longer_result = model_fields_to_annotated(Queryables.model_fields)
         self.assertGreater(len(queryables_peps_none), len(queryables_none_none))
         self.assertLess(len(queryables_peps_none), len(expected_longer_result))
         for key, queryable in queryables_peps_none.items():