Repository: ecmwf/earthkit-data
Pull request #636 (2025-03-05 19:14:32)

Enable creating list-of-dicts fieldlist without latlons

codecov-commenter: ## [Codecov](https://app.codecov.io/gh/ecmwf/earthkit-data/pull/636?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ecmwf) Report
All modified and coverable lines are covered by tests :white_check_mark:
> Project coverage is 90.02%. Comparing base [(`779a217`)](https://app.codecov.io/gh/ecmwf/earthkit-data/commit/779a217248e122212d96b7e593d90723a71d8206?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ecmwf) to head [(`7616300`)](https://app.codecov.io/gh/ecmwf/earthkit-data/commit/7616300b584a70e24beef1850b37dbff1e151744?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ecmwf).

<details><summary>Additional details and impacted files</summary>


```diff
@@           Coverage Diff            @@
##           develop     #636   +/-   ##
========================================
  Coverage    90.01%   90.02%           
========================================
  Files          158      158           
  Lines        12074    12086   +12     
  Branches       593      593           
========================================
+ Hits         10869    10881   +12     
  Misses        1023     1023           
  Partials       182      182           
```

</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/ecmwf/earthkit-data/pull/636?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ecmwf).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ecmwf).

<details><summary>🚀 New features to boost your workflow: </summary>

- ❄ [Test Analytics](https://docs.codecov.com/docs/test-analytics): Detect flaky tests, report on failures, and find test suite problems.
</details>

Patch:
diff --git a/docs/release_notes/version_0.13_updates.rst b/docs/release_notes/version_0.13_updates.rst
index f002d9a..e43bb50 100644
--- a/docs/release_notes/version_0.13_updates.rst
+++ b/docs/release_notes/version_0.13_updates.rst
@@ -52,4 +52,5 @@ New features
 - Refactored :ref:`data-sources-wekeo` and :ref:`data-sources-wekeocds` to use ``hda`` version 2 (:pr:`593`). The minimum ``hda`` version is now 2.22.
 - Added support for patterns with dates using timedelta as ``strftimedlta()`` for the :ref:`data-sources-file-pattern` source (:pr:`606`)
 - Enabled using string formatter for output file patterns in :ref:`new_grib_output <deprecated-new-grib-output>` and :ref:`GribOutput <deprecated-griboutput>` (:pr:`603`)
+- Enabled creating :ref:`data-sources-lod` fieldlists without latitudes/longitudes
 - Added :py:meth:`cpu` to the torch backend (:pr:`578`)
diff --git a/src/earthkit/data/sources/array_list.py b/src/earthkit/data/sources/array_list.py
index ec150af..144e614 100644
--- a/src/earthkit/data/sources/array_list.py
+++ b/src/earthkit/data/sources/array_list.py
@@ -37,7 +37,7 @@ class ArrayField(Field):
         if isinstance(metadata, dict):
             from earthkit.data.utils.metadata.dict import UserMetadata
 
-            metadata = UserMetadata(metadata, values=array)
+            metadata = UserMetadata(metadata, shape=array.shape)
 
         # TODO: this solution is questionable due to performance reasons
         if metadata is not None:
diff --git a/src/earthkit/data/sources/list_of_dicts.py b/src/earthkit/data/sources/list_of_dicts.py
index 3f4d328..1452665 100644
--- a/src/earthkit/data/sources/list_of_dicts.py
+++ b/src/earthkit/data/sources/list_of_dicts.py
@@ -33,7 +33,7 @@ class FieldlistFromDicts(Source):
             v = f["values"]
             if isinstance(v, list):
                 v = np.array(v)
-            fields.append(ArrayField(v, UserMetadata(f, values=v)))
+            fields.append(ArrayField(v, UserMetadata(f, shape=v.shape)))
         return SimpleFieldList(fields=fields)
 
 
diff --git a/src/earthkit/data/utils/metadata/dict.py b/src/earthkit/data/utils/metadata/dict.py
index fe91e70..dc8b7e0 100644
--- a/src/earthkit/data/utils/metadata/dict.py
+++ b/src/earthkit/data/utils/metadata/dict.py
@@ -32,7 +32,7 @@ def uniform_resolution(vals):
     return None
 
 
-def make_geography(metadata):
+def make_geography(metadata, values_shape=None):
     lat = metadata.get("latitudes", None)
     lon = metadata.get("longitudes", None)
     val = metadata.get("values")
@@ -46,25 +46,48 @@ def make_geography(metadata):
     if lat is None or lon is None:
         lat = metadata.get("distinctLatitudes", None)
         lon = metadata.get("distinctLongitudes", None)
-        lat = np.asarray(lat, dtype=float)
-        lon = np.asarray(lon, dtype=float)
-        if lat is None:
-            raise ValueError("No latitudes or distinctLatitudes found")
-        if lon is None:
-            raise ValueError("No longitudes or distinctLongitudes found")
-        if len(lat.shape) != 1:
-            raise ValueError(f"distinct latitudes must be 1D array! shape={lat.shape} unsupported")
-        if len(lon.shape) != 1:
-            raise ValueError(f"distinctLongitudes must be 1D array! shape={lon.shape} unsupported")
-        if lat.size * lon.size != val.size:
-            raise ValueError(
-                (
-                    "Distinct latitudes and longitudes do not match number of values. "
-                    f"Expected number=({lat.size * lon.size}), got={val.size}"
+        if values_shape is None:
+            if lat is None:
+                raise ValueError("No latitudes or distinctLatitudes found")
+            if lon is None:
+                raise ValueError("No longitudes or distinctLongitudes found")
+
+            lat = np.asarray(lat, dtype=float)
+            lon = np.asarray(lon, dtype=float)
+
+            if len(lat.shape) != 1:
+                raise ValueError(f"distinct latitudes must be 1D array! shape={lat.shape} unsupported")
+            if len(lon.shape) != 1:
+                raise ValueError(f"distinctLongitudes must be 1D array! shape={lon.shape} unsupported")
+            if lat.size * lon.size != val.size:
+                raise ValueError(
+                    (
+                        "Distinct latitudes and longitudes do not match number of values. "
+                        f"Expected number=({lat.size * lon.size}), got={val.size}"
+                    )
                 )
-            )
+            distinct = True
+        else:
+            if lat is not None and lon is not None:
+                lat = np.asarray(lat, dtype=float)
+                lon = np.asarray(lon, dtype=float)
+
+                if len(lat.shape) != 1:
+                    raise ValueError(f"distinct latitudes must be 1D array! shape={lat.shape} unsupported")
+                if len(lon.shape) != 1:
+                    raise ValueError(f"distinctLongitudes must be 1D array! shape={lon.shape} unsupported")
+                if lat.size * lon.size != val.size:
+                    raise ValueError(
+                        (
+                            "Distinct latitudes and longitudes do not match number of values. "
+                            f"Expected number=({lat.size * lon.size}), got={val.size}"
+                        )
+                    )
+                distinct = True
 
-        distinct = True
+            else:
+                lat = None
+                lon = None
     else:
         lat = np.asarray(lat, dtype=float)
         lon = np.asarray(lon, dtype=float)
@@ -79,10 +102,18 @@ def make_geography(metadata):
                 )
             distinct = True
 
-    assert lat is not None
-    assert lon is not None
+    no_latlon = lat is None or lon is None
+
+    if no_latlon:
+        assert values_shape is not None
+
+    if values_shape is None:
+        assert lat is not None
+        assert lon is not None
 
     if distinct:
+        assert not no_latlon
+
         dx = uniform_resolution(lon)
         dy = uniform_resolution(lat)
 
@@ -93,24 +124,27 @@ def make_geography(metadata):
         else:
             return DistinctLLGeography(metadata)
 
-    if lat.shape != lon.shape:
-        raise ValueError(f"latitudes and longitudes must have the same shape. {lat.shape} != {lon.shape}")
+    if no_latlon:
+        return UserGeography(metadata, shape=values_shape)
+    else:
+        if lat.shape != lon.shape:
+            raise ValueError(f"latitudes and longitudes must have the same shape. {lat.shape} != {lon.shape}")
 
-    if lat.size == val.size:
-        if lat.shape != val.shape:
-            shape = lat.shape if lat.ndim > val.ndim else val.shape
-        else:
-            shape = lat.shape
+        if lat.size == val.size:
+            if lat.shape != val.shape:
+                shape = lat.shape if lat.ndim > val.ndim else val.shape
+            else:
+                shape = lat.shape
 
-        return UserGeography(metadata, shape=shape)
+            return UserGeography(metadata, shape=shape)
 
-    else:
-        raise ValueError(
-            (
-                "latitudes and longitudes do not match number of values. "
-                f"Expected number=({lat.size * lon.size}), got={val.size}"
+        else:
+            raise ValueError(
+                (
+                    "latitudes and longitudes do not match number of values. "
+                    f"Expected number=({lat.size * lon.size}), got={val.size}"
+                )
             )
-        )
 
 
 class UserGeography(Geography):
@@ -282,8 +316,9 @@ class UserMetadata(Metadata):
 
     LS_KEYS = ["param", "level", "base_datetime", "valid_datetime", "step", "number"]
 
-    def __init__(self, d, values=None):
+    def __init__(self, d, shape=None, **kwargs):
         self._data = d
+        self._shape = shape
 
     def __len__(self):
         return len(self._data)
@@ -385,7 +420,7 @@ class UserMetadata(Metadata):
 
     @cached_property
     def geography(self):
-        return make_geography(self)
+        return make_geography(self, values_shape=self._shape)
 
     def override(self, *args, **kwargs):
         raise NotImplementedError("override is not implemented for UserMetadata")


Test patch:
diff --git a/tests/list_of_dicts/lod_fixtures.py b/tests/list_of_dicts/lod_fixtures.py
index 4d66957..8cfad03 100644
--- a/tests/list_of_dicts/lod_fixtures.py
+++ b/tests/list_of_dicts/lod_fixtures.py
@@ -166,6 +166,15 @@ def lod_ll_forecast_4():
     return _build_list(prototype)
 
 
+@pytest.fixture
+def lod_no_latlon():
+    prototype = {
+        "values": np.array([1, 2, 3, 4, 5, 6]),
+        "valid_datetime": "2018-08-01T09:00:00",
+    }
+    return _build_list(prototype)
+
+
 def build_lod_fieldlist(lod, mode):
     from earthkit.data import from_source
     from earthkit.data.indexing.fieldlist import SimpleFieldList
diff --git a/tests/list_of_dicts/test_lod_core.py b/tests/list_of_dicts/test_lod_core.py
index 3ca9e0d..0fd41c5 100644
--- a/tests/list_of_dicts/test_lod_core.py
+++ b/tests/list_of_dicts/test_lod_core.py
@@ -44,8 +44,8 @@ def test_lod_core(lod, mode, request):
     assert np.allclose(lon, lon_ref)
 
     val_ref = [[1, 2], [3, 4], [5, 6]]
-    print(ds[0].to_numpy())
-    print(ds.to_numpy())
+    # print(ds[0].to_numpy())
+    # print(ds.to_numpy())
 
     assert ds[0].to_numpy().shape == (3, 2)
     assert np.allclose(ds[0].to_numpy(), np.array(val_ref))
diff --git a/tests/list_of_dicts/test_lod_geography.py b/tests/list_of_dicts/test_lod_geography.py
index 87e1980..c828d93 100644
--- a/tests/list_of_dicts/test_lod_geography.py
+++ b/tests/list_of_dicts/test_lod_geography.py
@@ -149,6 +149,24 @@ def test_lod_geo_invalid(
         ds[0].shape
 
 
+@pytest.mark.parametrize("data", ["lod_no_latlon"])
+@pytest.mark.parametrize("mode", ["list-of-dicts", "loop"])
+def test_lod_no_latlon(
+    request,
+    data,
+    mode,
+):
+    ds = build_lod_fieldlist(request.getfixturevalue(data), mode)
+
+    assert len(ds) == 6
+    assert ds[0].shape == (6,)
+    assert ds[0].values.shape == (6,)
+    assert ds.values.shape == (6, 6)
+
+    with pytest.raises(ValueError):
+        ds[0].to_latlon()
+
+
 if __name__ == "__main__":
     from earthkit.data.testing import main