Repository: ecmwf/earthkit-data
Pull request #256 (2023-11-13 11:58:40)

Add to_decoded method to fieldlist
This PR adds the `to_decoded` method to fieldlists. It can convert a fieldlist into a NumpyFieldList in such a way that all the values will be stored as a single numpy array.

![image](https://github.com/ecmwf/earthkit-data/assets/9033020/b04b6576-abaa-439d-aec1-f64f06a729e6)



Patch:
diff --git a/docs/api.rst b/docs/api.rst
index 16fc7ea..d5f064d 100644
--- a/docs/api.rst
+++ b/docs/api.rst
@@ -11,6 +11,7 @@ Fields
 - :py:class:`~data.core.fieldlist.Field`
 - :py:class:`~data.core.fieldlist.FieldList`
 
+
 Metadata
 ----------
 
@@ -18,6 +19,11 @@ Metadata
 - :py:class:`~data.core.metadata.RawMetadata`
 - :py:class:`~data.readers.grib.metadata.GribMetadata`
 
+Numpy fields
+---------------
+- :py:class:`~data.sources.numpy_list.NumpyField`
+- :py:class:`~data.sources.numpy_list.NumpyFieldList`
+
 GRIB
 -------
 
@@ -35,6 +41,8 @@ CSV
 
 - :py:class:`~data.readers.csv.CSVReader`
 
+
+
 Other
 --------
 
diff --git a/docs/skip_api_rules.py b/docs/skip_api_rules.py
index 3e9f4fb..7d905e0 100644
--- a/docs/skip_api_rules.py
+++ b/docs/skip_api_rules.py
@@ -83,6 +83,26 @@ _skip_methods = {
         "merger",
         "source",
     ],
+    "data.source.numpy_list.NumpyField": ["merge", "mutate"],
+    "data.source.numpy_list.NumpyFieldList": [
+        "cache_file",
+        "dataset",
+        "from_dict",
+        "from_mask",
+        "from_multi",
+        "from_slice",
+        "full",
+        "graph",
+        "ignore",
+        "merge",
+        "mutate",
+        "new_mask_index",
+        "parent",
+        "scaled",
+        "settings",
+        "statistics",
+        "xarray_open_dataset_kwargs",
+    ],
 }
 
 
@@ -100,6 +120,8 @@ def _skip_api_items(app, what, name, obj, skip, options):
         "data.readers.grib.codes",
         "data.readers.grib.index",
         "data.readers.csv",
+        "data.sources",
+        "data.sources.numpy_list",
         "data.utils",
         "data.utils.bbox",
     ]:
@@ -113,6 +135,8 @@ def _skip_api_items(app, what, name, obj, skip, options):
         "data.readers.grib",
         "data.readers.grib.index",
         "data.readers.csv",
+        "data.sources",
+        "data.sources.numpy_list",
         "data.utils",
         "data.utils.bbox",
     ]:
@@ -128,6 +152,8 @@ def _skip_api_items(app, what, name, obj, skip, options):
         "data.readers.grib.index.GribFieldList",
         "data.readers.grib.metadata.GribMetadata",
         "data.readers.csv.CSVReader",
+        "data.sources.numpy_list.NumpyField",
+        "data.sources.numpy_list.NumpyFieldList",
         "data.utils.bbox.BoundingBox",
     ]:
         skip = True
diff --git a/earthkit/data/core/fieldlist.py b/earthkit/data/core/fieldlist.py
index a4e3b8a..0471595 100644
--- a/earthkit/data/core/fieldlist.py
+++ b/earthkit/data/core/fieldlist.py
@@ -7,6 +7,7 @@
 # nor does it submit to any jurisdiction.
 #
 
+import math
 from abc import abstractmethod
 from collections import defaultdict
 
@@ -24,7 +25,7 @@ class Field(Base):
 
     @abstractmethod
     def _values(self, dtype=None):
-        r"""Return the values stored in the field as a 1D ndarray.
+        r"""Return the values stored in the field as an ndarray.
 
         Parameters
         ----------
@@ -44,7 +45,11 @@ class Field(Base):
     @property
     def values(self):
         r"""ndarray: Get the values stored in the field as a 1D ndarray."""
-        return self._values()
+        v = self._values()
+        if len(v.shape) != 1:
+            n = math.prod(v.shape)
+            return v.reshape(n)
+        return v
 
     def _make_metadata(self):
         r"""Create a field metadata object."""
@@ -75,13 +80,18 @@ class Field(Base):
             Field values
 
         """
-        values = self.values
-        if not flatten:
-            # values = self.values.reshape(self.shape)
-            values = self._values(dtype=dtype).reshape(self.shape)
-        if dtype is not None:
-            values = values.astype(dtype)
-        return values
+        v = self._values(dtype=dtype)
+        shape = self._required_shape(flatten)
+        if shape != v.shape:
+            return v.reshape(shape)
+        return v
+
+    def _required_shape(self, flatten):
+        return self.shape if not flatten else (math.prod(self.shape),)
+
+    def _array_matches(self, array, flatten=False, dtype=None):
+        shape = self._required_shape(flatten)
+        return shape == array.shape and (dtype is None or dtype == array.dtype)
 
     def data(self, keys=("lat", "lon", "value"), flatten=False, dtype=None):
         r"""Return the values and/or the geographical coordinates for each grid point.
@@ -150,9 +160,10 @@ class Field(Base):
                 raise ValueError(f"data: invalid argument: {k}")
 
         r = [_keys[k](dtype=dtype) for k in keys]
-        if not flatten:
-            shape = self.shape
+        shape = self._required_shape(flatten)
+        if shape != r[0].shape:
             r = [x.reshape(shape) for x in r]
+
         if len(r) == 1:
             return r[0]
         else:
@@ -193,8 +204,8 @@ class Field(Base):
         x = self._metadata.geography.x(dtype=dtype)
         y = self._metadata.geography.y(dtype=dtype)
         if x is not None and y is not None:
-            if not flatten:
-                shape = self.shape
+            shape = self._required_shape(flatten)
+            if shape != x.shape:
                 x = x.reshape(shape)
                 y = y.reshape(shape)
             return dict(x=x, y=y)
@@ -522,6 +533,11 @@ class Field(Base):
             result[name] = self._metadata.get(name, None)
         return result
 
+    def to_decoded(self, **kwargs):
+        from earthkit.data.sources.numpy_list import NumpyField
+
+        return NumpyField(self.to_numpy(**kwargs), self.metadata())
+
 
 class FieldList(Index):
     r"""Represents a list of :obj:`Field` \s."""
@@ -1178,3 +1194,27 @@ class FieldList(Index):
         """
         for s in self:
             s.write(f)
+
+    def to_decoded(self, **kwargs):
+        r"""Convert the FieldList into a :class:`NumpyFieldList` storing all the
+        data in memory.
+
+        In the resulting object each field is represented by an ndarray storing the
+        field values and a :class:`MetaData` object holding the field metadata. The
+        shape and dtype of the ndarray is controlled by the ``kwargs``.
+
+        Internally, the generated :class:`NumpyFieldList` will always store all the
+        field values in a single ndarray.
+
+        Parameters
+        ----------
+        **kwargs: dict, optional
+            Keyword arguments passed to :obj:`to_numpy`
+
+        Returns
+        -------
+        :class:`NumpyFieldList`
+
+        """
+        md = [f.metadata() for f in self]
+        return self.from_numpy(self.to_numpy(**kwargs), md)
diff --git a/earthkit/data/readers/grib/__init__.py b/earthkit/data/readers/grib/__init__.py
index 70e2f3d..5d14250 100644
--- a/earthkit/data/readers/grib/__init__.py
+++ b/earthkit/data/readers/grib/__init__.py
@@ -41,6 +41,6 @@ def stream_reader(source, stream, magic=None, deeper_check=False):
         from .memory import GribFieldListInMemory, GribStreamReader
 
         r = GribStreamReader(stream)
-        if source.batch_size == 0:
+        if not source.group_by and source.batch_size == 0:
             r = GribFieldListInMemory(source, r)
         return r
diff --git a/earthkit/data/readers/grib/codes.py b/earthkit/data/readers/grib/codes.py
index 4dcf21a..ae6616b 100644
--- a/earthkit/data/readers/grib/codes.py
+++ b/earthkit/data/readers/grib/codes.py
@@ -267,11 +267,6 @@ class GribField(Field):
     def _values(self, dtype=None):
         return self.handle.get_values(dtype=dtype)
 
-    # @property
-    # def values(self):
-    #     r"""ndarray: Gets the values stored in the GRIB field as a 1D ndarray."""
-    #     return self.handle.get_values()
-
     @property
     def offset(self):
         r"""number: Gets the offset (in bytes) of the GRIB field within the GRIB file."""
diff --git a/earthkit/data/readers/grib/xarray.py b/earthkit/data/readers/grib/xarray.py
index a73794b..b14aa63 100644
--- a/earthkit/data/readers/grib/xarray.py
+++ b/earthkit/data/readers/grib/xarray.py
@@ -60,7 +60,7 @@ class XarrayMixIn:
 
     def to_xarray(self, **kwargs):
         """
-        Converts the fieldset into an xarray DataSet using :xref:`cfgrib`.
+        Convert the FieldList into an xarray DataSet using :xref:`cfgrib`.
 
         Parameters
         ----------
diff --git a/earthkit/data/readers/netcdf.py b/earthkit/data/readers/netcdf.py
index eb5a5ea..4cd0062 100644
--- a/earthkit/data/readers/netcdf.py
+++ b/earthkit/data/readers/netcdf.py
@@ -430,18 +430,10 @@ class XArrayField(Field):
         return self.to_xarray().to_numpy()
 
     def _values(self, dtype=None):
-        return self._to_numpy().flatten()
-
-    def to_numpy(self, flatten=False, dtype=None):
-        values = self._to_numpy()
-        if not flatten:
-            values = values.reshape(self.shape)
+        if dtype is None:
+            return self._to_numpy()
         else:
-            values = values.flatten()
-        if dtype is not None:
-            values = values.astype(dtype)
-
-        return values
+            return self._to_numpy().astype(dtype, copy=False)
 
 
 class XArrayFieldListCore(FieldList):
diff --git a/earthkit/data/sources/numpy_list.py b/earthkit/data/sources/numpy_list.py
index 66b9d8a..99cac1e 100644
--- a/earthkit/data/sources/numpy_list.py
+++ b/earthkit/data/sources/numpy_list.py
@@ -20,6 +20,16 @@ LOG = logging.getLogger(__name__)
 
 
 class NumpyField(Field):
+    r"""Represents a field consisting of an ndarray and metadata object.
+
+    Parameters
+    ----------
+    array: ndarray
+        Array storing the values of the field
+    metadata: :class:`Metadata`
+        Metadata object describing the field metadata.
+    """
+
     def __init__(self, array, metadata):
         self._array = array
         super().__init__(metadata=metadata)
@@ -31,7 +41,13 @@ class NumpyField(Field):
         if dtype is None:
             return self._array
         else:
-            return self._array.astype(dtype)
+            return self._array.astype(dtype, copy=False)
+
+    def to_decoded(self, **kwargs):
+        if self._array_matches(self._array, **kwargs):
+            return self
+        else:
+            return NumpyField(self.to_numpy(**kwargs), self.metadata())
 
     def __repr__(self):
         return f"{self.__class__.__name__}()"
@@ -108,6 +124,12 @@ class NumpyFieldListCore(PandasMixIn, XarrayMixIn, FieldList):
     def __repr__(self):
         return f"{self.__class__.__name__}(fields={len(self)})"
 
+    def to_decoded(self, **kwargs):
+        if self[0]._array_matches(self._array[0], **kwargs):
+            return self
+        else:
+            return type(self)(self.to_numpy(**kwargs), self._metadata)
+
 
 class MultiUnwindMerger:
     def __init__(self, sources):
@@ -142,6 +164,18 @@ class ListMerger:
 
 
 class NumpyFieldList(NumpyFieldListCore):
+    r"""Represents a list of :obj:`NumpyField <data.sources.numpy_list.NumpyField>`\ s.
+
+    The preferred way to create a NumpyFieldList is to use either the
+    static :obj:`from_numpy` method or the :obj:`to_decoded` method.
+
+    See Also
+    --------
+    from_numpy
+    to_decoded
+
+    """
+
     def _getitem(self, n):
         if isinstance(n, int):
             return NumpyField(self._array[n], self._metadata[n])
diff --git a/earthkit/data/sources/stream.py b/earthkit/data/sources/stream.py
index 5fcffff..f4f6d18 100644
--- a/earthkit/data/sources/stream.py
+++ b/earthkit/data/sources/stream.py
@@ -25,6 +25,9 @@ class StreamMemorySource(MemoryBaseSource):
     def __iter__(self):
         return iter(self._reader)
 
+    def to_decoded(self, **kwargs):
+        return self._reader.to_decoded(**kwargs)
+
 
 class StreamSource(Source):
     def __init__(self, stream, group_by=None, **kwargs):
@@ -39,20 +42,22 @@ class StreamSource(Source):
             try:
                 self._group_by = list(self._group_by)
             except Exception:
-                raise TypeError(f"unsupported types in `group_by`={self._group_by}")
+                raise TypeError(f"unsupported types in group_by={self._group_by}")
 
         if self._group_by and not all([isinstance(x, str) for x in self._group_by]):
             raise TypeError(f"`group_by`={self._group_by} must contain str values")
 
         if self._group_by and "batch_size" in kwargs:
             raise TypeError(
-                "got an invalid keyword argument. `batch_size` cannot be used when `group_by` is set"
+                "got an invalid keyword argument. batch_size cannot be used when group_by is set"
             )
 
         self._batch_size = kwargs.pop("batch_size", 1)
+        if self._batch_size is None:
+            self._batch_size = 1
 
         if self._batch_size < 0:
-            raise ValueError(f"`batch_size`={self._batch_size} cannot be negative")
+            raise ValueError(f"batch_size={self._batch_size} cannot be negative")
         if kwargs:
             raise TypeError(f"got invalid keyword argument(s): {list(kwargs.keys())}")
 


Test patch:
diff --git a/tests/grib/test_grib_stream.py b/tests/grib/test_grib_stream.py
index f094be3..35ef77e 100644
--- a/tests/grib/test_grib_stream.py
+++ b/tests/grib/test_grib_stream.py
@@ -9,6 +9,7 @@
 # nor does it submit to any jurisdiction.
 #
 
+import numpy as np
 import pytest
 
 from earthkit.data import from_source
@@ -58,29 +59,110 @@ def test_grib_from_stream_group_by(group_by):
         for i, f in enumerate(fs):
             assert len(f) == 3
             assert f.metadata(("param", "level")) == ref[i]
+            assert f.to_decoded() is not f
 
         # stream consumed, no data is available
         assert sum([1 for _ in fs]) == 0
 
 
+@pytest.mark.parametrize(
+    "decode_kwargs,expected_shape",
+    [
+        ({}, (3, 7, 12)),
+        (None, (3, 7, 12)),
+        (None, (3, 7, 12)),
+        ({"flatten": False}, (3, 7, 12)),
+        ({"flatten": True}, (3, 84)),
+    ],
+)
+def test_grib_from_stream_group_by_decode(decode_kwargs, expected_shape):
+    group_by = "level"
+    with open(earthkit_examples_file("test6.grib"), "rb") as stream:
+        ds = from_source("stream", stream, group_by=group_by)
+
+        # no fieldlist methods are available on a StreamSource
+        with pytest.raises(TypeError):
+            len(ds)
+
+        ref = [
+            [("t", 1000), ("u", 1000), ("v", 1000)],
+            [("t", 850), ("u", 850), ("v", 850)],
+        ]
+
+        if decode_kwargs is None:
+            decode_kwargs = {}
+
+        for i, f in enumerate(ds):
+            df = f.to_decoded(**decode_kwargs)
+            assert len(df) == 3
+            assert df.metadata(("param", "level")) == ref[i]
+            assert df._array.shape == expected_shape
+            assert df.to_numpy(**decode_kwargs).shape == expected_shape
+            assert df.to_decoded(**decode_kwargs) is df
+
+        # stream consumed, no data is available
+        assert sum([1 for _ in ds]) == 0
+
+
 def test_grib_from_stream_single_batch():
     with open(earthkit_examples_file("test6.grib"), "rb") as stream:
-        fs = from_source("stream", stream)
+        ds = from_source("stream", stream)
 
-        # no methods are available
+        # no fieldlist methods are available
         with pytest.raises(TypeError):
-            len(fs)
+            len(ds)
 
-        ref = ["t", "u", "v", "t", "u", "v"]
-        val = []
-        for f in fs:
-            v = f.metadata("param")
-            val.append(v)
+        ref = [
+            ("t", 1000),
+            ("u", 1000),
+            ("v", 1000),
+            ("t", 850),
+            ("u", 850),
+            ("v", 850),
+        ]
 
-        assert val == ref
+        for i, f in enumerate(ds):
+            assert f.metadata(("param", "level")) == ref[i], i
 
         # stream consumed, no data is available
-        assert sum([1 for _ in fs]) == 0
+        assert sum([1 for _ in ds]) == 0
+
+
+@pytest.mark.parametrize(
+    "decode_kwargs,expected_shape",
+    [
+        ({}, (7, 12)),
+        (None, (7, 12)),
+        (None, (7, 12)),
+        ({"flatten": False}, (7, 12)),
+        ({"flatten": True}, (84,)),
+    ],
+)
+def test_grib_from_stream_single_batch_decode(decode_kwargs, expected_shape):
+    with open(earthkit_examples_file("test6.grib"), "rb") as stream:
+        ds = from_source("stream", stream)
+
+        ref = [
+            ("t", 1000),
+            ("u", 1000),
+            ("v", 1000),
+            ("t", 850),
+            ("u", 850),
+            ("v", 850),
+        ]
+
+        if decode_kwargs is None:
+            decode_kwargs = {}
+
+        for i, f in enumerate(ds):
+            df = f.to_decoded(**decode_kwargs)
+            assert df.metadata(("param", "level")) == ref[i], i
+            assert df._array.shape == expected_shape, i
+            assert df.to_numpy(**decode_kwargs).shape == expected_shape, i
+            assert df.to_decoded(**decode_kwargs) is df, i
+
+        # stream consumed, no data is available
+        assert sum([1 for _ in ds]) == 0
 
 
 def test_grib_from_stream_multi_batch():
@@ -100,17 +182,115 @@ def test_grib_from_stream_multi_batch():
         assert sum([1 for _ in fs]) == 0
 
 
+@pytest.mark.parametrize(
+    "decode_kwargs,expected_shape",
+    [
+        ({}, (2, 7, 12)),
+        (None, (2, 7, 12)),
+        (None, (2, 7, 12)),
+        ({"flatten": False}, (2, 7, 12)),
+        (
+            {"flatten": True},
+            (
+                2,
+                84,
+            ),
+        ),
+    ],
+)
+def test_grib_from_stream_multi_batch_decode(decode_kwargs, expected_shape):
+    with open(earthkit_examples_file("test6.grib"), "rb") as stream:
+        ds = from_source("stream", stream, batch_size=2)
+
+        ref = [
+            [("t", 1000), ("u", 1000)],
+            [("v", 1000), ("t", 850)],
+            [("u", 850), ("v", 850)],
+        ]
+
+        if decode_kwargs is None:
+            decode_kwargs = {}
+
+        for i, f in enumerate(ds):
+            df = f.to_decoded(**decode_kwargs)
+            assert df.metadata(("param", "level")) == ref[i], i
+            assert df._array.shape == expected_shape, i
+            assert df.to_numpy(**decode_kwargs).shape == expected_shape, i
+            assert df.to_decoded(**decode_kwargs) is df, i
+
+        # stream consumed, no data is available
+        assert sum([1 for _ in ds]) == 0
+
+
 def test_grib_from_stream_in_memory():
     with open(earthkit_examples_file("test6.grib"), "rb") as stream:
-        fs = from_source("stream", stream, batch_size=0)
+        ds = from_source(
+            "stream",
+            stream,
+            batch_size=0,
+        )
 
-        assert len(fs) == 6
+        assert len(ds) == 6
+
+        expected_shape = (6, 7, 12)
+        ref = ["t", "u", "v", "t", "u", "v"]
+        val = []
+
+        # iteration
+        for f in ds:
+            v = f.metadata("param")
+            val.append(v)
+
+        assert val == ref, "iteration"
+
+        # metadata
+        val = []
+        val = ds.metadata("param")
+        assert val == ref, "method"
+
+        # data
+        assert ds.to_numpy().shape == expected_shape
+
+        ref = np.array(
+            [
+                272.56417847,
+                -6.28688049,
+                7.83348083,
+                272.53916931,
+                -4.89837646,
+                8.66096497,
+            ]
+        )
+
+        vals = ds.to_numpy()[:, 0, 0]
+        assert np.allclose(vals, ref)
+
+
+@pytest.mark.parametrize(
+    "decode_kwargs,expected_shape",
+    [
+        ({}, (6, 7, 12)),
+        ({"flatten": False}, (6, 7, 12)),
+        ({"flatten": True}, (6, 84)),
+    ],
+)
+def test_grib_from_stream_in_memory_decode(decode_kwargs, expected_shape):
+    with open(earthkit_examples_file("test6.grib"), "rb") as stream:
+        ds_s = from_source(
+            "stream",
+            stream,
+            batch_size=0,
+        )
+
+        ds = ds_s.to_decoded(**decode_kwargs)
+
+        assert len(ds) == 6
 
         ref = ["t", "u", "v", "t", "u", "v"]
         val = []
 
         # iteration
-        for f in fs:
+        for f in ds:
             v = f.metadata("param")
             val.append(v)
 
@@ -118,9 +298,32 @@ def test_grib_from_stream_in_memory():
 
         # metadata
         val = []
-        val = fs.metadata("param")
+        val = ds.metadata("param")
         assert val == ref, "method"
 
+        # data
+        assert ds.to_numpy(**decode_kwargs).shape == expected_shape
+
+        ref = np.array(
+            [
+                272.56417847,
+                -6.28688049,
+                7.83348083,
+                272.53916931,
+                -4.89837646,
+                8.66096497,
+            ]
+        )
+
+        if len(expected_shape) == 3:
+            vals = ds.to_numpy(**decode_kwargs)[:, 0, 0]
+        else:
+            vals = ds.to_numpy(**decode_kwargs)[:, 0]
+
+        assert np.allclose(vals, ref)
+        assert ds._array.shape == expected_shape
+        assert ds.to_decoded(**decode_kwargs) is ds
+
 
 def test_grib_save_when_loaded_from_stream():
     with open(earthkit_examples_file("test6.grib"), "rb") as stream:
diff --git a/tests/numpy_fs/numpy_fs_fixtures.py b/tests/numpy_fs/numpy_fs_fixtures.py
index 02d49a7..213ac73 100644
--- a/tests/numpy_fs/numpy_fs_fixtures.py
+++ b/tests/numpy_fs/numpy_fs_fixtures.py
@@ -93,6 +93,55 @@ def check_numpy_fs(ds, ds_input, md_full):
         assert r[2].metadata("param") == "u"
 
 
+def check_numpy_fs_decoded(ds, ds_input, md_full, flatten=False, dtype=None):
+    assert len(ds_input) in [1, 2, 3]
+    assert len(ds) == len(md_full)
+    assert ds.metadata("param") == md_full
+
+    np_kwargs = {"flatten": flatten, "dtype": dtype}
+
+    assert np.allclose(
+        ds[0].to_numpy(**np_kwargs), ds_input[0][0].to_numpy(**np_kwargs)
+    )
+
+    assert ds.to_numpy(**np_kwargs).shape == ds_input[0].to_numpy(**np_kwargs).shape
+    assert ds._array.shape == ds_input[0].to_numpy(**np_kwargs).shape
+
+    # check slice
+    r = ds[1]
+    assert r.metadata("param") == "msl"
+
+    if len(ds_input) > 1:
+        r = ds[1:3]
+        assert len(r) == 2
+        assert r.metadata("param") == ["msl", "t"]
+        assert r[0].metadata("param") == "msl"
+        assert r[1].metadata("param") == "t"
+        assert np.allclose(
+            r[0].to_numpy(**np_kwargs), ds_input[0][1].to_numpy(**np_kwargs)
+        )
+        assert np.allclose(
+            r[1].to_numpy(**np_kwargs), ds_input[1][0].to_numpy(**np_kwargs)
+        )
+
+        # check sel
+        r = ds.sel(shortName="msl")
+        assert len(r) == 1
+        assert r.metadata("shortName") == ["msl"]
+        assert r[0].metadata("param") == "msl"
+        assert np.allclose(
+            r[0].to_numpy(**np_kwargs), ds_input[0][1].to_numpy(**np_kwargs)
+        )
+
+    if len(ds_input) == 3:
+        r = ds[1:13:4]
+        assert len(r) == 3
+        assert r.metadata("param") == ["msl", "t", "u"]
+        assert r[0].metadata("param") == "msl"
+        assert r[1].metadata("param") == "t"
+        assert r[2].metadata("param") == "u"
+
+
 def check_save_to_disk(ds, len_ref, meta_ref):
     tmp = temp_file()
     ds.save(tmp.path)
diff --git a/tests/numpy_fs/test_numpy_fs.py b/tests/numpy_fs/test_numpy_fs.py
index c5e4105..17ec100 100644
--- a/tests/numpy_fs/test_numpy_fs.py
+++ b/tests/numpy_fs/test_numpy_fs.py
@@ -22,7 +22,7 @@ from earthkit.data.testing import earthkit_examples_file
 
 here = os.path.dirname(__file__)
 sys.path.insert(0, here)
-from numpy_fs_fixtures import check_numpy_fs  # noqa: E402
+from numpy_fs_fixtures import check_numpy_fs, check_numpy_fs_decoded  # noqa: E402
 
 
 def test_numpy_fs_grib_single_field():
@@ -113,6 +113,40 @@ def test_numpy_fs_grib_from_list_of_arrays_bad():
         _ = FieldList.from_numpy([v], md)
 
 
+@pytest.mark.parametrize(
+    "kwargs",
+    [
+        {},
+        {"dtype": np.float32},
+        {"flatten": False},
+        {"flatten": True},
+        {"flatten": True, "dtype": np.float32},
+    ],
+)
+def test_numpy_fs_grib_from_to_decoded(kwargs):
+    ds = from_source("file", earthkit_examples_file("test.grib"))
+    md_full = ds.metadata("param")
+    assert len(ds) == 2
+
+    r = ds.to_decoded(**kwargs)
+    check_numpy_fs_decoded(r, [ds], md_full, **kwargs)
+
+
+def test_numpy_fs_grib_from_to_decoded_repeat():
+    ds = from_source("file", earthkit_examples_file("test.grib"))
+    md_full = ds.metadata("param")
+    assert len(ds) == 2
+
+    kwargs = {}
+    r = ds.to_decoded(**kwargs)
+    check_numpy_fs_decoded(r, [ds], md_full, **kwargs)
+
+    kwargs = {"flatten": True, "dtype": np.float32}
+    r1 = r.to_decoded(**kwargs)
+    assert r1 is not r
+    check_numpy_fs_decoded(r1, [ds], md_full, **kwargs)
+
+
 if __name__ == "__main__":
     from earthkit.data.testing import main