Repository: ecmwf/earthkit-data (python)

Issue:
Use ecCodes header only cloning for GRIB numpy fieldlists
### Is your feature request related to a problem? Please describe.

The header only cloning is new feature in ecCodes and will be available in these versions:

- eccodes-python - v1.7.0 or higher
- eccodes - v2.34.0 or higher

With this the metadata part in a NumpyField would only contain a message with a cloned header (no data part) significantly reducing the size of the field.

### Describe the solution you'd like

_No response_

### Describe alternatives you've considered

_No response_

### Additional context

_No response_

### Organisation

ECMWF

Pull request description:
Feature/header only clone
Fixes #278 

Patch:
diff --git a/earthkit/data/readers/grib/metadata.py b/earthkit/data/readers/grib/metadata.py
index d591244..db1fbe9 100644
--- a/earthkit/data/readers/grib/metadata.py
+++ b/earthkit/data/readers/grib/metadata.py
@@ -253,7 +253,7 @@ class GribMetadata(Metadata):
 
     def override(self, *args, **kwargs):
         d = dict(*args, **kwargs)
-        handle = self._handle.clone()
+        handle = self._handle.clone(headers_only=True)
         handle.set_multiple(d)
         return GribMetadata(handle)
 
@@ -382,11 +382,13 @@ class RestrictedGribMetadata(GribMetadata):
         "numberOfMissing",
         "numberOfCodedValues",
         "bitmapPresent",
+        "bitsPerValue",
         "offsetValuesBy",
         "packingError",
         "referenceValue",
         "referenceValueError",
         "unpackedError",
+        "values",
     ]
     INTERNAL_NAMESPACES = ["statistics"]
 
diff --git a/earthkit/data/utils/message.py b/earthkit/data/utils/message.py
index bf3a37a..6d840a3 100644
--- a/earthkit/data/utils/message.py
+++ b/earthkit/data/utils/message.py
@@ -7,6 +7,7 @@
 # nor does it submit to any jurisdiction.
 #
 
+import functools
 import json
 import logging
 import os
@@ -33,6 +34,44 @@ except Exception:
     pass
 
 
+class EccodesFeatures:
+    def __init__(self):
+        v = eccodes.codes_get_version_info()
+        try:
+            self._version = tuple([int(x) for x in v["eccodes"].split(".")])
+        except Exception:
+            self._version = (0, 0, 0)
+
+        try:
+            self._py_version = tuple([int(x) for x in v["bindings"].split(".")])
+        except Exception:
+            self._py_version = (0, 0, 0)
+
+        LOG.debug(f"ecCodes versions: {self.versions}")
+
+    def check_clone_kwargs(self, **kwargs):
+        if not (self._py_version >= (1, 7, 0) and self._version >= (2, 34, 0)):
+            kwargs = dict(**kwargs)
+            kwargs.pop("headers_only", None)
+        return kwargs
+
+    @property
+    def versions(self):
+        return f"ecCodes: {self._version} eccodes-python: {self._py_version}"
+
+
+ECC_FEATURES = EccodesFeatures()
+
+
+def check_clone_kwargs(func):
+    @functools.wraps(func)
+    def wrapped(*args, **kwargs):
+        kwargs = ECC_FEATURES.check_clone_kwargs(**kwargs)
+        return func(*args, **kwargs)
+
+    return wrapped
+
+
 class CodesMessagePositionIndex:
     VERSION = 1
 
@@ -147,8 +186,9 @@ class CodesHandle(eccodes.Message):
     def get_long(self, name):
         return self.get(name, ktype=int)
 
-    def clone(self):
-        return self._from_raw_handle(eccodes.codes_clone(self._handle))
+    @check_clone_kwargs
+    def clone(self, **kwargs):
+        return self._from_raw_handle(eccodes.codes_clone(self._handle, **kwargs))
 
     def set_multiple(self, values):
         try:
diff --git a/earthkit/data/writers/grib.py b/earthkit/data/writers/grib.py
index 3209110..4c064af 100644
--- a/earthkit/data/writers/grib.py
+++ b/earthkit/data/writers/grib.py
@@ -27,7 +27,7 @@ class GribWriter(Writer):
         check_nans: bool
             Replace nans in ``values`` with GRIB missing values when writing to``f``.
         """
-        handle = metadata._handle
+        handle = metadata._handle.clone()
         if check_nans:
             import numpy as np
 


Test patch:
diff --git a/tests/grib/test_grib_metadata.py b/tests/grib/test_grib_metadata.py
index 850beda..660ed61 100644
--- a/tests/grib/test_grib_metadata.py
+++ b/tests/grib/test_grib_metadata.py
@@ -193,7 +193,7 @@ def test_grib_get_long_array_1(mode):
     assert pl[72] == 312
 
 
-@pytest.mark.parametrize("mode", ["file", "numpy_fs"])
+@pytest.mark.parametrize("mode", ["file"])
 def test_grib_get_double_array_values_1(mode):
     f = load_file_or_numpy_fs("test_single.grib", mode, folder="data")
 
@@ -212,7 +212,7 @@ def test_grib_get_double_array_values_1(mode):
     )
 
 
-@pytest.mark.parametrize("mode", ["file", "numpy_fs"])
+@pytest.mark.parametrize("mode", ["file"])
 def test_grib_get_double_array_values_18(mode):
     f = load_file_or_numpy_fs("tuv_pl.grib", mode)
     v = f.metadata("values")
diff --git a/tests/grib/test_grib_summary.py b/tests/grib/test_grib_summary.py
index 8cfdaf0..11c5b67 100644
--- a/tests/grib/test_grib_summary.py
+++ b/tests/grib/test_grib_summary.py
@@ -203,20 +203,20 @@ def test_grib_ls_keys(mode):
 
     # default keys
     # positive num (=head)
-    df = f.ls(n=2, keys=["shortName", "bitsPerValue", "gridType"])
+    df = f.ls(n=2, keys=["shortName", "bottomLevel", "gridType"])
     ref = {
         "shortName": {0: "t", 1: "u"},
-        "bitsPerValue": {0: 4, 1: 4},
+        "bottomLevel": {0: 1000, 1: 1000},
         "gridType": {0: "regular_ll", 1: "regular_ll"},
     }
 
     assert ref == df.to_dict()
 
     # negative num (=tail)
-    df = f.ls(n=-2, keys=["shortName", "bitsPerValue", "gridType"])
+    df = f.ls(n=-2, keys=["shortName", "bottomLevel", "gridType"])
     ref = {
         "shortName": {0: "u", 1: "v"},
-        "bitsPerValue": {0: 4, 1: 4},
+        "bottomLevel": {0: 300, 1: 300},
         "gridType": {0: "regular_ll", 1: "regular_ll"},
     }
 
diff --git a/tests/numpy_fs/test_numpy_fs_metadata.py b/tests/numpy_fs/test_numpy_fs_metadata.py
index 8a9c810..52672a1 100644
--- a/tests/numpy_fs/test_numpy_fs_metadata.py
+++ b/tests/numpy_fs/test_numpy_fs_metadata.py
@@ -12,7 +12,6 @@
 import os
 import sys
 
-import numpy as np
 import pytest
 
 here = os.path.dirname(__file__)
@@ -38,6 +37,7 @@ def test_numpy_fs_values_metadata():
         "const",
         "bitmapPresent",
         "numberOfMissing",
+        "values",
     ]
     for k in keys:
         assert ds[0].metadata(k, default=None) is None, k
@@ -56,14 +56,6 @@ def test_numpy_fs_values_metadata_internal():
     for k, v in keys.items():
         assert ds[0].metadata(k) == v, k
 
-    keys = {
-        "grib.min": 262.7802734375,
-        "grib.max": 315.4599609375,
-    }
-
-    for k, v in keys.items():
-        assert np.isclose(ds[0].metadata(k), v), k
-
 
 def test_numpy_fs_metadata_keys():
     ds, _ = load_numpy_fs(1)
diff --git a/tests/numpy_fs/test_numpy_fs_write.py b/tests/numpy_fs/test_numpy_fs_write.py
index 5dbefe9..59f6191 100644
--- a/tests/numpy_fs/test_numpy_fs_write.py
+++ b/tests/numpy_fs/test_numpy_fs_write.py
@@ -128,7 +128,7 @@ def test_numpy_fs_grib_write_generating_proc_id():
 
     r1 = FieldList.from_numpy([v1, v2], [md1, md2])
 
-    # save to disk: using generatingProcessIdentifier=255 (default)
+    # save to disk
     with temp_file() as tmp:
         r1.save(tmp)
         assert os.path.exists(tmp)
@@ -140,6 +140,9 @@ def test_numpy_fs_grib_write_generating_proc_id():
             150,
         ]
 
+        assert np.allclose(r_tmp.values[0], v1)
+        assert np.allclose(r_tmp.values[1], v2)
+
 
 if __name__ == "__main__":
     from earthkit.data.testing import main