Repository: ecmwf/earthkit-data (python)

Issue:
Hide value related GRIB metadata in NumpyFieldList
### Current situation

If we create a new NumpyFieldList with the following code:

```python
# f is a GribField
vals_new = f.values + 1
md_new = f.metadata().override(shortName="pt")
ds_new = FieldList.from_numpy(vals_new, md_new)
```

then the each new NumpyField consists of a numpy array (`vals_new`) and a metadata object, which stores a GRIB handle cloned from the handle in the original GRIB field but the values in it are not updated with `vals_new`. See #203 for an explanation.

As a result, there is an inconsistency between the various Field methods related to the values:

```python
# these use the numpy array
ds_new[0].values
ds_new[0].to_numpy()
ds_new[0].data()

# these use the metadata object
ds_new[0].metadata("max")
ds_new[0].metadata("bitmapPresent")
ds_new[0].metadata("numberOfMissing")
ds_new[0].metadata(namespace="statistics")
ds_new[0].dump()
ds_new[0].dump(namespace="statistics")
```

```python
>>> ds_new[0].values.max()
234.12
>>> ds_new[0].metadata("max")
233.12
```
### The proposed solution

These GRIB keys are in fact not metadata keys but rather related to the actual values in the field. Moreover we can get the correct value related stats by inspecting the numpy array. So the right decision is to hide them in a NumpyFieldLists. 

The expected behaviour:

```python
# this throws a KeyError
ds_new[0].metadata("max")
ds_new[0].metadata()["max"]

# these return None
ds_new[0].metadata("max", None)
ds_new[0].metadata().get("max")
```

The "statistics" namespace will be hidden, whether we use the `namspace` kwarg in `metadata()` or we use the `dump()`. E.g. this is how the `dump()` output would look like:

<img width="532" alt="image" src="https://github.com/ecmwf/earthkit-data/assets/9033020/22f8cfc5-e41e-45f9-b415-94ffbe7450ab">



Pull request description:
Feature/numpy fieldlist value related metadata
Fixes #215 

Patch:
diff --git a/earthkit/data/core/fieldlist.py b/earthkit/data/core/fieldlist.py
index 2fd9620..a4e3b8a 100644
--- a/earthkit/data/core/fieldlist.py
+++ b/earthkit/data/core/fieldlist.py
@@ -431,10 +431,10 @@ class Field(Base):
             default = kwargs.pop("default", None)
 
             r = [
-                self._metadata._get_key(
+                self._metadata.get(
                     k,
-                    astype=kt,
                     default=default,
+                    astype=kt,
                     raise_on_missing=raise_on_missing,
                     **kwargs,
                 )
diff --git a/earthkit/data/core/metadata.py b/earthkit/data/core/metadata.py
index cd1c9ee..990b964 100644
--- a/earthkit/data/core/metadata.py
+++ b/earthkit/data/core/metadata.py
@@ -7,7 +7,6 @@
 # nor does it submit to any jurisdiction.
 #
 
-
 from abc import ABCMeta, abstractmethod
 
 from earthkit.data.core.constants import DATETIME
@@ -27,6 +26,13 @@ class Metadata(metaclass=ABCMeta):
 
     """
 
+    DATA_FORMAT = None
+    NAMESPACES = []
+    LS_KEYS = []
+    DESCRIBE_KEYS = []
+    INDEX_KEYS = []
+    CUSTOM_KEYS = [DATETIME]
+
     def __iter__(self):
         """Return an iterator over the metadata keys."""
         return iter(self.keys())
@@ -34,6 +40,7 @@ class Metadata(metaclass=ABCMeta):
     @abstractmethod
     def __len__(self):
         r"""Return the number of metadata entries."""
+        pass
 
     def __getitem__(self, key):
         r"""Return the value for ``key``.
@@ -43,7 +50,7 @@ class Metadata(metaclass=ABCMeta):
         KeyError
             When ``key`` is not available.
         """
-        return self._get_key(key)
+        return self.get(key, raise_on_missing=True)
 
     @abstractmethod
     def __contains__(self, key):
@@ -57,7 +64,7 @@ class Metadata(metaclass=ABCMeta):
 
     @abstractmethod
     def keys(self):
-        r"""Return a new view of the metadata keys.
+        r"""Return the metadata keys.
 
         Returns
         -------
@@ -68,7 +75,7 @@ class Metadata(metaclass=ABCMeta):
 
     @abstractmethod
     def items(self):
-        r"""Return a new view of the metadata items.
+        r"""Return the metadata items.
 
         Returns
         -------
@@ -77,19 +84,7 @@ class Metadata(metaclass=ABCMeta):
         """
         pass
 
-    def get(self, key, default=None):
-        r"""Return the value for ``key``.
-
-        Parameters
-        ----------
-        key: str
-            Metadata key
-        default:
-            When ``key`` is not found ``default`` is returned.
-        """
-        return self._get_key(key, default=default, raise_on_missing=False)
-
-    def _get_key(self, key, astype=None, default=None, raise_on_missing=True):
+    def get(self, key, default=None, *, astype=None, raise_on_missing=False):
         r"""Return the value for ``key``.
 
         Parameters
@@ -99,6 +94,10 @@ class Metadata(metaclass=ABCMeta):
         default: value
             Specify the default value for ``key``. Returned when ``key``
             is not found or its value is a missing value and raise_on_missing is ``False``.
+        astype: type as str, int or float
+            Return/access type for ``key``. When it is supported ``astype`` is passed to the
+            underlying metadata accessor as an option. Otherwise the value is
+            cast to ``astype`` after it is taken from the accessor.
         raise_on_missing: bool
             When it is True raises an exception if ``key`` is not found or
             it has a missing value.
@@ -116,32 +115,32 @@ class Metadata(metaclass=ABCMeta):
             a missing value.
 
         """
-        # check for custom keys
-        is_custom, v = self._get_custom_key(
-            key, default=default, raise_on_missing=raise_on_missing
-        )
-        if is_custom:
-            return v
-
-        return self._get_internal_key(
-            key, astype=astype, default=default, raise_on_missing=raise_on_missing
-        )
+        if self._is_custom_key(key):
+            return self._get_custom_key(
+                key, default=default, astype=astype, raise_on_missing=raise_on_missing
+            )
+        else:
+            return self._get(
+                key, default=default, astype=astype, raise_on_missing=raise_on_missing
+            )
 
     @abstractmethod
-    def _get_internal_key(self, key, astype=None, default=None, raise_on_missing=True):
+    def _get(self, key, astype=None, default=None, raise_on_missing=False):
         pass
 
-    def _get_custom_key(self, key, default=None, raise_on_missing=True):
-        try:
-            if key == DATETIME:
-                if key not in self:
-                    return True, self.datetime().get("valid_time")
-        except Exception as e:
-            if not raise_on_missing:
-                return True, default
-            else:
-                raise KeyError(f"{key}, reason={e}")
-        return False, None
+    def _is_custom_key(self, key):
+        return key in self.CUSTOM_KEYS and key not in self
+
+    def _get_custom_key(self, key, default=None, raise_on_missing=True, **kwargs):
+        if self._is_custom_key(key):
+            try:
+                if key == DATETIME:
+                    return self._valid_datetime()
+            except Exception as e:
+                if not raise_on_missing:
+                    return default
+                else:
+                    raise KeyError(f"{key}, reason={e}")
 
     @abstractmethod
     def override(self, *args, **kwargs):
@@ -169,7 +168,7 @@ class Metadata(metaclass=ABCMeta):
         -------
         list of str
         """
-        return []
+        return self.NAMESPACES
 
     def as_namespace(self, namespace=None):
         r"""Return all the keys/values from a namespace.
@@ -195,15 +194,33 @@ class Metadata(metaclass=ABCMeta):
         return None
 
     def datetime(self):
-        r"""Return the date and time.
+        r"""Return the date and time of the field.
 
         Returns
         -------
-        dict of datatime.datetime or None
-            Dict with items "base_time" and "valid_time". None if
-            date and time are not available.
+        dict of datatime.datetime
+            Dict with items "base_time" and "valid_time".
+
+
+        >>> import earthkit.data
+        >>> ds = earthkit.data.from_source("file", "tests/data/t_time_series.grib")
+        >>> ds[4].datetime()
+        {'base_time': datetime.datetime(2020, 12, 21, 12, 0),
+        'valid_time': datetime.datetime(2020, 12, 21, 18, 0)}
+
         """
-        return None
+        return {
+            "base_time": self._base_datetime(),
+            "valid_time": self._valid_datetime(),
+        }
+
+    @abstractmethod
+    def _base_datetime(self):
+        pass
+
+    @abstractmethod
+    def _valid_datetime(self):
+        pass
 
     @property
     def geography(self):
@@ -215,15 +232,28 @@ class Metadata(metaclass=ABCMeta):
 
     def ls_keys(self):
         r"""Return the keys to be used with the :meth:`ls` method."""
-        return []
+        return self.LS_KEYS
 
     def describe_keys(self):
         r"""Return the keys to be used with the :meth:`describe` method."""
-        return []
+        return self.DESCRIBE_KEYS
 
     def index_keys(self):
         r"""Return the keys to be used with the :meth:`indices` method."""
-        return []
+        return self.INDEX_KEYS
+
+    def data_format(self):
+        r"""Return the underlying data format.
+
+        Returns
+        -------
+        str
+
+        """
+        return self.DATA_FORMAT
+
+    def _hide_internal_keys(self):
+        return self
 
 
 class RawMetadata(Metadata):
@@ -262,19 +292,18 @@ class RawMetadata(Metadata):
     def __init__(self, *args, **kwargs):
         self._d = dict(*args, **kwargs)
 
+    def override(self, *args, **kwargs):
+        d = dict(**self._d)
+        d.update(*args, **kwargs)
+        return RawMetadata(d)
+
     def __len__(self):
         return len(self._d)
 
-    def keys(self):
-        return self._d.keys()
-
-    def items(self):
-        return self._d.items()
-
     def __contains__(self, key):
         return key in self._d
 
-    def _get_internal_key(self, key, astype=None, default=None, raise_on_missing=True):
+    def _get(self, key, astype=None, default=None, raise_on_missing=False):
         if not raise_on_missing:
             v = self._d.get(key, default)
         else:
@@ -287,10 +316,20 @@ class RawMetadata(Metadata):
                 return None
         return v
 
-    def override(self, *args, **kwargs):
-        d = dict(**self._d)
-        d.update(*args, **kwargs)
-        return RawMetadata(d)
+    def keys(self):
+        return self._d.keys()
+
+    def items(self):
+        return self._d.items()
+
+    def _base_datetime(self):
+        return None
+
+    def _valid_datetime(self):
+        return None
+
+    def as_namespace(self, namespace):
+        return {}
 
     def __repr__(self):
         return f"{self.__class__.__name__}({self._d.__repr__()})"
diff --git a/earthkit/data/readers/grib/metadata.py b/earthkit/data/readers/grib/metadata.py
index 1441150..af4ea5c 100644
--- a/earthkit/data/readers/grib/metadata.py
+++ b/earthkit/data/readers/grib/metadata.py
@@ -199,6 +199,8 @@ class GribMetadata(Metadata):
         "vertical",
     ]
 
+    DATA_FORMAT = "grib"
+
     __handle_type = None
 
     def __init__(self, handle):
@@ -221,32 +223,18 @@ class GribMetadata(Metadata):
         return GribMetadata.__handle_type
 
     def __len__(self):
-        return sum(map(lambda i: 1, self._handle.keys()))
+        return sum(map(lambda i: 1, self.keys()))
 
     def __contains__(self, key):
         return self._handle.__contains__(key)
 
     def keys(self):
-        r"""Return a new view of the metadata keys.
-
-        Returns
-        -------
-        Iterable of str
-
-        """
         return self._handle.keys()
 
     def items(self):
-        r"""Return a new view of the metadata items.
-
-        Returns
-        -------
-        Iterable of :obj:`(key,value)` pairs
-
-        """
         return self._handle.items()
 
-    def _get_internal_key(self, key, astype=None, default=None, raise_on_missing=False):
+    def _get(self, key, astype=None, default=None, raise_on_missing=False):
         def _key_name(key):
             if key == "param":
                 key = "shortName"
@@ -260,22 +248,10 @@ class GribMetadata(Metadata):
 
         return self._handle.get(_key_name(key), ktype=astype, **_kwargs)
 
-    def override(self, *args, **kwargs):
-        r"""Change the metadata values and return a new object.
-
-        :obj:`override` accepts another :obj:`Metadata` or a dict or
-        an iterable of key/value pairs (as tuples or other iterables of length two).
-        If keyword arguments are specified, the metadata is then updated with those
-        key/value pairs.
+    def _is_custom_key(self, key):
+        return key in self.CUSTOM_KEYS
 
-        Examples
-        --------
-        >>> other = RawMetadata({"key1": 1, "key2": 2})
-        >>> m1 = m.override(other)
-        >>> m1 = m.override({"key1": 1, "key2": 2})
-        >>> m1 = m.override([("key1", 1), ("key2", 2)])
-        >>> m1 = m.override(key1=1, key2=2)
-        """
+    def override(self, *args, **kwargs):
         d = dict(*args, **kwargs)
         handle = self._handle.clone()
         handle.set_multiple(d)
@@ -311,36 +287,6 @@ class GribMetadata(Metadata):
             self._geo = GribFieldGeography(self)
         return self._geo
 
-    def namespaces(self):
-        r"""Return the available namespaces.
-
-        Returns
-        -------
-        list of str
-        """
-        return self.NAMESPACES
-
-    def datetime(self):
-        r"""Return the date and time of the field.
-
-        Returns
-        -------
-        dict of datatime.datetime
-            Dict with items "base_time" and "valid_time".
-
-
-        >>> import earthkit.data
-        >>> ds = earthkit.data.from_source("file", "tests/data/t_time_series.grib")
-        >>> ds[4].datetime()
-        {'base_time': datetime.datetime(2020, 12, 21, 12, 0),
-        'valid_time': datetime.datetime(2020, 12, 21, 18, 0)}
-
-        """
-        return {
-            "base_time": self._base_datetime(),
-            "valid_time": self._valid_datetime(),
-        }
-
     def _base_datetime(self):
         date = self.get("date", None)
         time = self.get("time", None)
@@ -393,27 +339,133 @@ class GribMetadata(Metadata):
         from earthkit.data.utils.summary import format_namespace_dump
 
         namespace = self.NAMESPACES if namespace is all else [namespace]
-        r = [
-            {
-                "title": ns if ns else "default",
-                "data": self.as_namespace(ns),
-                "tooltip": f"Keys in the ecCodes {ns} namespace",
-            }
-            for ns in namespace
-        ]
+        r = []
+        for ns in namespace:
+            v = self.as_namespace(ns)
+            if v:
+                r.append(
+                    {
+                        "title": ns if ns else "default",
+                        "data": v,
+                        "tooltip": f"Keys in the ecCodes {ns} namespace",
+                    }
+                )
 
         return format_namespace_dump(
             r, selected="parameter", details=self.__class__.__name__, **kwargs
         )
 
-    def ls_keys(self):
-        r"""Return the keys to be used with the :meth:`ls` method."""
-        return self.LS_KEYS
+    def _hide_internal_keys(self):
+        return RestrictedGribMetadata(self)
+
+
+# TODO: this is a temporary solution
+class RestrictedGribMetadata(GribMetadata):
+    """Hide internal keys and namespaces in GRIB metadata"""
+
+    EKD_NAMESPACE = "grib"
+    INTERNAL_KEYS = [
+        "min",
+        "max",
+        "avg",
+        "sd",
+        "skew",
+        "kurt",
+        "const",
+        "isConstant",
+        "numberOfMissing",
+        "numberOfCodedValues",
+        "bitmapPresent",
+        "offsetValuesBy",
+        "packingError",
+        "referenceValue",
+        "referenceValueError",
+        "unpackedError",
+    ]
+    INTERNAL_NAMESPACES = ["statistics"]
 
-    def describe_keys(self):
-        r"""Return the keys to be used with the :meth:`describe` method."""
-        return self.DESCRIBE_KEYS
+    def __init__(self, md):
+        super().__init__(md._handle)
 
-    def index_keys(self):
-        r"""Return the keys to be used with the :meth:`indices` method."""
-        return self.INDEX_KEYS
+    def __len__(self):
+        if self.INTERNAL_KEYS:
+            return len(self.keys())
+        else:
+            return super().__len__()
+
+    def _is_internal(self, key):
+        ns, _, name = key.partition(".")
+        if name == "":
+            name = key
+            ns = ""
+
+        if ns == self.EKD_NAMESPACE:
+            return False
+        else:
+            return name in self.INTERNAL_KEYS
+
+    def __contains__(self, key):
+        if self.INTERNAL_KEYS:
+            return not self._is_internal(key) and super().__contains__(key)
+        else:
+            return super().__contains__(key)
+
+    def keys(self):
+        if self.INTERNAL_KEYS:
+            r = []
+            for k in super().keys():
+                if k not in self.INTERNAL_KEYS:
+                    r.append(k)
+            return r
+        else:
+            return super().keys()
+
+    def items(self):
+        if self.INTERNAL_KEYS:
+            r = {}
+            for k, v in super().items():
+                if k not in self.INTERNAL_KEYS:
+                    r[k] = v
+            return r.items()
+        else:
+            return super().items()
+
+    def get(self, key, default=None, *, astype=None, raise_on_missing=False):
+        ns, _, name = key.partition(".")
+        if name == "":
+            name = key
+            ns = ""
+
+        if ns == self.EKD_NAMESPACE:
+            key = name
+        else:
+            if name in self.INTERNAL_KEYS:
+                if raise_on_missing:
+                    raise KeyError(key)
+                else:
+                    return default
+
+        return super().get(
+            key, default=default, astype=astype, raise_on_missing=raise_on_missing
+        )
+
+    def namespaces(self):
+        if self.INTERNAL_NAMESPACES:
+            return [
+                x for x in super().namespaces() if x not in self.INTERNAL_NAMESPACES
+            ]
+        else:
+            return super().namespaces()
+
+    def as_namespace(self, namespace):
+        if namespace in self.INTERNAL_NAMESPACES:
+            return {}
+
+        r = super().as_namespace(namespace)
+        for k in list(r.keys()):
+            if k in self.INTERNAL_KEYS:
+                del r[k]
+        return r
+
+    def _hide_internal_keys(self):
+        return self
diff --git a/earthkit/data/readers/netcdf.py b/earthkit/data/readers/netcdf.py
index a94b64f..eb5a5ea 100644
--- a/earthkit/data/readers/netcdf.py
+++ b/earthkit/data/readers/netcdf.py
@@ -329,6 +329,8 @@ class XArrayFieldGeography(Geography):
 
 
 class XArrayMetadata(RawMetadata):
+    LS_KEYS = ["variable", "level", "time", "units"]
+
     def __init__(self, field):
         if not isinstance(field, XArrayField):
             raise TypeError(
@@ -357,13 +359,11 @@ class XArrayMetadata(RawMetadata):
             )
         return self._geo
 
-    def datetime(self):
-        t = to_datetime(self._field.time)
-        return {"base_time": t, "valid_time": t}
+    def _base_datetime(self):
+        return self._valid_datetime()
 
-    def ls_keys(self):
-        LS_KEYS = ["variable", "level", "time", "units"]
-        return LS_KEYS
+    def _valid_datetime(self):
+        return to_datetime(self._field.time)
 
 
 class XArrayField(Field):
diff --git a/earthkit/data/sources/constants.py b/earthkit/data/sources/constants.py
index 06212c1..8f4e402 100644
--- a/earthkit/data/sources/constants.py
+++ b/earthkit/data/sources/constants.py
@@ -34,14 +34,11 @@ class ConstantMetadata(RawMetadata):
     def geography(self):
         return self._geo
 
-    def datetime(self):
-        return {
-            "base_time": None,
-            "valid_time": datetime.datetime.fromisoformat(self["valid_datetime"]),
-        }
-
-    def ls_keys(self):
-        return self.LS_KEYS
+    def _base_datetime(self):
+        return None
+
+    def _valid_datetime(self):
+        return datetime.datetime.fromisoformat(self["valid_datetime"])
 
 
 class ConstantMaker:
diff --git a/earthkit/data/sources/numpy_list.py b/earthkit/data/sources/numpy_list.py
index 3502a84..66b9d8a 100644
--- a/earthkit/data/sources/numpy_list.py
+++ b/earthkit/data/sources/numpy_list.py
@@ -13,7 +13,6 @@ import numpy as np
 
 from earthkit.data.core.fieldlist import Field, FieldList
 from earthkit.data.core.index import MaskIndex, MultiIndex
-from earthkit.data.core.metadata import Metadata
 from earthkit.data.readers.grib.pandas import PandasMixIn
 from earthkit.data.readers.grib.xarray import XarrayMixIn
 
@@ -51,10 +50,6 @@ class NumpyFieldListCore(PandasMixIn, XarrayMixIn, FieldList):
         if not isinstance(self._metadata, list):
             self._metadata = [self._metadata]
 
-        for md in self._metadata:
-            if not isinstance(md, Metadata):
-                raise TypeError("metadata must be a subclass of MetaData")
-
         if isinstance(self._array, np.ndarray):
             if self._array.shape[0] != len(self._metadata):
                 # we have a single array and a single metadata
@@ -87,6 +82,9 @@ class NumpyFieldListCore(PandasMixIn, XarrayMixIn, FieldList):
         else:
             raise TypeError("array must be an ndarray or a list of ndarrays")
 
+        # hide internal metadata related to values
+        self._metadata = [md._hide_internal_keys() for md in self._metadata]
+
         super().__init__(*args, **kwargs)
 
     def _shape_match(self, shape1, shape2):
diff --git a/earthkit/data/writers/__init__.py b/earthkit/data/writers/__init__.py
index 0b93d7e..527570b 100644
--- a/earthkit/data/writers/__init__.py
+++ b/earthkit/data/writers/__init__.py
@@ -27,7 +27,7 @@ class Writer(metaclass=ABCMeta):
 
 
 def write(f, values, metadata, **kwargs):
-    x = _writers(metadata.__class__.__name__)
+    x = _writers(metadata.data_format())
     c = x()
     c.write(f, values, metadata, **kwargs)
 
@@ -46,7 +46,7 @@ def _writers(method_name):
                     module = import_module(f".{name}", package=__name__)
                     if hasattr(module, "Writer"):
                         w = getattr(module, "Writer")
-                        _WRITERS[w.METADATA_TYPE] = w
+                        _WRITERS[w.DATA_FORMAT] = w
                         _WRITERS[name] = w
                 except Exception:
                     LOG.exception("Error loading writer %s", name)
diff --git a/earthkit/data/writers/grib.py b/earthkit/data/writers/grib.py
index 9ca6b79..6e0fa39 100644
--- a/earthkit/data/writers/grib.py
+++ b/earthkit/data/writers/grib.py
@@ -11,7 +11,7 @@ from . import Writer
 
 
 class GribWriter(Writer):
-    METADATA_TYPE = "GribMetadata"
+    DATA_FORMAT = "grib"
 
     def write(self, f, values, metadata, check_nans=True):
         handle = metadata._handle


Test patch:
diff --git a/tests/grib/test_grib_metadata.py b/tests/grib/test_grib_metadata.py
index 5c90786..6edc1f4 100644
--- a/tests/grib/test_grib_metadata.py
+++ b/tests/grib/test_grib_metadata.py
@@ -129,9 +129,9 @@ def test_grib_metadata_astype_18(mode, key, astype, expected_value):
 @pytest.mark.parametrize(
     "key,expected_value",
     [
-        ("max", 307.18560791015625),
-        ("max:d", 307.18560791015625),
-        ("max:float", 307.18560791015625),
+        ("latitudeOfFirstGridPointInDegrees", 90.0),
+        ("latitudeOfFirstGridPointInDegrees:d", 90.0),
+        ("latitudeOfFirstGridPointInDegrees:float", 90.0),
     ],
 )
 def test_grib_metadata_double_1(mode, key, expected_value):
@@ -145,35 +145,15 @@ def test_grib_metadata_double_1(mode, key, expected_value):
 @pytest.mark.parametrize(
     "key",
     [
-        ("max"),
-        ("max:d"),
-        ("max:float"),
+        ("latitudeOfFirstGridPointInDegrees"),
+        ("latitudeOfFirstGridPointInDegrees:d"),
+        ("latitudeOfFirstGridPointInDegrees:float"),
     ],
 )
 def test_grib_metadata_double_18(mode, key):
     f = load_file_or_numpy_fs("tuv_pl.grib", mode)
 
-    ref = [
-        320.5641784667969,
-        17.713119506835938,
-        11.833480834960938,
-        304.53916931152344,
-        27.10162353515625,
-        12.660964965820312,
-        287.26531982421875,
-        28.145523071289062,
-        15.6385498046875,
-        271.8430633544922,
-        36.74000549316406,
-        15.009902954101562,
-        264.00323486328125,
-        46.213775634765625,
-        23.949615478515625,
-        250.6531524658203,
-        58.45549011230469,
-        36.92034912109375,
-    ]
-
+    ref = [90.0] * 18
     r = f.metadata(key)
     np.testing.assert_allclose(r, ref, 0.001)
 
@@ -182,33 +162,14 @@ def test_grib_metadata_double_18(mode, key):
 @pytest.mark.parametrize(
     "key,astype",
     [
-        ("max", None),
-        ("max", float),
+        ("latitudeOfFirstGridPointInDegrees", None),
+        ("latitudeOfFirstGridPointInDegrees", float),
     ],
 )
 def test_grib_metadata_double_astype_18(mode, key, astype):
     f = load_file_or_numpy_fs("tuv_pl.grib", mode)
 
-    ref = [
-        320.5641784667969,
-        17.713119506835938,
-        11.833480834960938,
-        304.53916931152344,
-        27.10162353515625,
-        12.660964965820312,
-        287.26531982421875,
-        28.145523071289062,
-        15.6385498046875,
-        271.8430633544922,
-        36.74000549316406,
-        15.009902954101562,
-        264.00323486328125,
-        46.213775634765625,
-        23.949615478515625,
-        250.6531524658203,
-        58.45549011230469,
-        36.92034912109375,
-    ]
+    ref = [90.0] * 18
 
     r = f.metadata(key, astype=astype)
     np.testing.assert_allclose(r, ref, 0.001)
@@ -435,7 +396,7 @@ def test_grib_metadata_missing_key(mode):
     assert v == 0
 
 
-@pytest.mark.parametrize("mode", ["file", "numpy_fs"])
+@pytest.mark.parametrize("mode", ["file"])
 def test_grib_metadata_namespace(mode):
     f = load_file_or_numpy_fs("test6.grib", mode)
 
diff --git a/tests/grib/test_grib_summary.py b/tests/grib/test_grib_summary.py
index e32f09b..8cfdaf0 100644
--- a/tests/grib/test_grib_summary.py
+++ b/tests/grib/test_grib_summary.py
@@ -359,7 +359,7 @@ def test_grib_tail_num(mode):
     assert ref == df.to_dict()
 
 
-@pytest.mark.parametrize("mode", ["file", "numpy_fs"])
+@pytest.mark.parametrize("mode", ["file"])
 def test_grib_dump(mode):
     f = load_file_or_numpy_fs("test6.grib", mode)
 
diff --git a/tests/numpy_fs/numpy_fs_fixtures.py b/tests/numpy_fs/numpy_fs_fixtures.py
index 37f1e90..02d49a7 100644
--- a/tests/numpy_fs/numpy_fs_fixtures.py
+++ b/tests/numpy_fs/numpy_fs_fixtures.py
@@ -41,6 +41,17 @@ def load_numpy_fs(num):
     return (*ds, md)
 
 
+def load_numpy_fs_file(fname):
+    ds_in = from_source("file", earthkit_examples_file(fname))
+    md = ds_in.metadata("param")
+
+    ds = FieldList.from_numpy(
+        ds_in.values, [m.override(edition=1) for m in ds_in.metadata()]
+    )
+
+    return (ds, md)
+
+
 def check_numpy_fs(ds, ds_input, md_full):
     assert len(ds_input) in [1, 2, 3]
 
@@ -48,6 +59,11 @@ def check_numpy_fs(ds, ds_input, md_full):
     assert ds.metadata("param") == md_full
     assert np.allclose(ds[0].values, ds_input[0][0].values)
 
+    # # values metadata
+    # keys = ["min", "max"]
+    # for k in keys:
+    #     assert np.isclose(ds[0].metadata(k), ds_input[0][0].metadata(k))
+
     # check slice
     r = ds[1]
     assert r.metadata("param") == "msl"
diff --git a/tests/numpy_fs/test_numpy_fs_metadata.py b/tests/numpy_fs/test_numpy_fs_metadata.py
new file mode 100644
index 0000000..ad505e3
--- /dev/null
+++ b/tests/numpy_fs/test_numpy_fs_metadata.py
@@ -0,0 +1,174 @@
+#!/usr/bin/env python3
+
+# (C) Copyright 2020 ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+#
+
+import os
+import sys
+
+import numpy as np
+import pytest
+
+here = os.path.dirname(__file__)
+sys.path.insert(0, here)
+from numpy_fs_fixtures import load_numpy_fs, load_numpy_fs_file  # noqa: E402
+
+# Note: Almost all grib metadata tests are also run for numpyfs.
+# See grib/test_grib_metadata.py
+
+
+def test_numpy_fs_values_metadata():
+    ds, _ = load_numpy_fs(1)
+
+    # values metadata
+    keys = [
+        "min",
+        "max",
+        "avg",
+        "ds",
+        "skew",
+        "kurt",
+        "isConstant",
+        "const",
+        "bitmapPresent",
+        "numberOfMissing",
+    ]
+    for k in keys:
+        assert ds[0].metadata(k, default=None) is None, k
+        with pytest.raises(KeyError):
+            ds[0].metadata(k)
+
+
+def test_numpy_fs_values_metadata_internal():
+    ds, _ = load_numpy_fs(1)
+
+    keys = {
+        "shortName": "2t",
+        "grib.shortName": "2t",
+    }
+
+    for k, v in keys.items():
+        assert ds[0].metadata(k) == v, k
+
+    keys = {
+        "grib.min": 262.7802734375,
+        "grib.max": 315.4599609375,
+    }
+
+    for k, v in keys.items():
+        assert np.isclose(ds[0].metadata(k), v), k
+
+
+def test_numpy_fs_metadata_keys():
+    ds, _ = load_numpy_fs(1)
+
+    num = 183
+    md = ds[0].metadata()
+    assert len(md) == num
+
+    keys = md.keys()
+    assert len(keys) == num
+
+    for k in md.keys():
+        assert isinstance(k, str)
+        assert k != ""
+        break
+
+    items = md.items()
+    assert len(items) == num
+
+    for k, v in md.items():
+        assert isinstance(k, str)
+        assert k != ""
+        assert v is not None
+        break
+
+    assert "max" not in md
+    assert "statistics.max" not in md
+    assert "validityDate" in md
+
+
+def test_numpy_fs_metadata_namespace():
+    f, _ = load_numpy_fs_file("tuv_pl.grib")
+
+    r = f[0].metadata(namespace="vertical")
+    ref = {"level": 1000, "typeOfLevel": "isobaricInhPa"}
+    assert r == ref
+
+    r = f[0].metadata(namespace=["vertical", "time"])
+    ref = {
+        "vertical": {"typeOfLevel": "isobaricInhPa", "level": 1000},
+        "time": {
+            "dataDate": 20180801,
+            "dataTime": 1200,
+            "stepUnits": 1,
+            "stepType": "instant",
+            "stepRange": "0",
+            "startStep": 0,
+            "endStep": 0,
+            "validityDate": 20180801,
+            "validityTime": 1200,
+        },
+    }
+    assert r == ref
+
+    r = f[0].metadata(namespace=None)
+    assert isinstance(r, dict)
+    assert len(r) == 177
+    assert r["level"] == 1000
+    assert r["stepType"] == "instant"
+
+    r = f[0].metadata(namespace=[None])
+    assert isinstance(r, dict)
+    assert len(r) == 177
+    assert r["level"] == 1000
+    assert r["stepType"] == "instant"
+
+    r = f[0].metadata(namespace="")
+    assert isinstance(r, dict)
+    assert len(r) == 177
+    assert r["level"] == 1000
+    assert r["stepType"] == "instant"
+
+    r = f[0].metadata(namespace=[""])
+    assert isinstance(r, dict)
+    assert len(r) == 177
+    assert r["level"] == 1000
+    assert r["stepType"] == "instant"
+
+    ref = {
+        "geography",
+        "vertical",
+        "time",
+        "parameter",
+        "mars",
+        "ls",
+        "default",
+    }
+    r = f[0].metadata(namespace=all)
+    assert isinstance(r, dict)
+    assert set(r.keys()) == ref
+
+    r = f[0].metadata(namespace=[all])
+    assert isinstance(r, dict)
+    assert set(r.keys()) == ref
+
+    with pytest.raises(ValueError) as excinfo:
+        r = f[0].metadata("level", namespace=["vertical", "time"])
+    assert "must be a str when key specified" in str(excinfo.value)
+
+    with pytest.raises(ValueError) as excinfo:
+        r = f[0].metadata("level", namespace=["vertical", "time"])
+    assert "must be a str when key specified" in str(excinfo.value)
+
+
+if __name__ == "__main__":
+    from earthkit.data.testing import main
+
+    main(__file__)
diff --git a/tests/numpy_fs/test_numpy_fs_summary.py b/tests/numpy_fs/test_numpy_fs_summary.py
new file mode 100644
index 0000000..8b1646f
--- /dev/null
+++ b/tests/numpy_fs/test_numpy_fs_summary.py
@@ -0,0 +1,176 @@
+#!/usr/bin/env python3
+
+# (C) Copyright 2020 ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+#
+
+import os
+import sys
+
+here = os.path.dirname(__file__)
+sys.path.insert(0, here)
+from numpy_fs_fixtures import load_numpy_fs_file  # noqa: E402
+
+# Note: Almost all grib metadata tests are also run for numpyfs.
+# See grib/test_grib_summary.py
+
+
+def test_numpy_fs_dump():
+    f, _ = load_numpy_fs_file("test6.grib")
+
+    namespaces = (
+        "default",
+        "geography",
+        "ls",
+        "mars",
+        "parameter",
+        "time",
+        "vertical",
+    )
+
+    # default
+    r = f[0].dump(_as_raw=True)
+    ref = [
+        {
+            "title": "ls",
+            "data": {
+                "edition": 1,
+                "centre": "ecmf",
+                "typeOfLevel": "isobaricInhPa",
+                "level": 1000,
+                "dataDate": 20180801,
+                "stepRange": "0",
+                "dataType": "an",
+                "shortName": "t",
+                "packingType": "grid_simple",
+                "gridType": "regular_ll",
+            },
+            "tooltip": "Keys in the ecCodes ls namespace",
+        },
+        {
+            "title": "geography",
+            "data": {
+                # "bitmapPresent": 0,
+                "latitudeOfFirstGridPointInDegrees": 90.0,
+                "longitudeOfFirstGridPointInDegrees": 0.0,
+                "latitudeOfLastGridPointInDegrees": -90.0,
+                "longitudeOfLastGridPointInDegrees": 330.0,
+                "iScansNegatively": 0,
+                "jScansPositively": 0,
+                "jPointsAreConsecutive": 0,
+                "jDirectionIncrementInDegrees": 30.0,
+                "iDirectionIncrementInDegrees": 30.0,
+                "gridType": "regular_ll",
+            },
+            "tooltip": "Keys in the ecCodes geography namespace",
+        },
+        {
+            "title": "mars",
+            "data": {
+                "domain": "g",
+                "levtype": "pl",
+                "levelist": 1000,
+                "date": 20180801,
+                "time": 1200,
+                "step": 0,
+                "param": "t",
+                "class": "od",
+                "type": "an",
+                "stream": "oper",
+                "expver": "0001",
+            },
+            "tooltip": "Keys in the ecCodes mars namespace",
+        },
+        {
+            "title": "parameter",
+            "data": {
+                "centre": "ecmf",
+                "paramId": 130,
+                "units": "K",
+                "name": "Temperature",
+                "shortName": "t",
+            },
+            "tooltip": "Keys in the ecCodes parameter namespace",
+        },
+        {
+            "title": "time",
+            "data": {
+                "dataDate": 20180801,
+                "dataTime": 1200,
+                "stepUnits": 1,
+                "stepType": "instant",
+                "stepRange": "0",
+                "startStep": 0,
+                "endStep": 0,
+                "validityDate": 20180801,
+                "validityTime": 1200,
+            },
+            "tooltip": "Keys in the ecCodes time namespace",
+        },
+        {
+            "title": "vertical",
+            "data": {"typeOfLevel": "isobaricInhPa", "level": 1000},
+            "tooltip": "Keys in the ecCodes vertical namespace",
+        },
+    ]
+
+    assert len(r) == len(namespaces)
+    assert isinstance(r, list)
+    for d in r:
+        ns = d["title"]
+        assert ns in namespaces
+        if ns not in ("default", "statistics"):
+            assert d == [x for x in ref if x["title"] == ns][0], ns
+
+    # a namespace
+    r = f[0].dump(namespace="mars", _as_raw=True)
+    ref = [
+        {
+            "title": "mars",
+            "data": {
+                "domain": "g",
+                "levtype": "pl",
+                "levelist": 1000,
+                "date": 20180801,
+                "time": 1200,
+                "step": 0,
+                "param": "t",
+                "class": "od",
+                "type": "an",
+                "stream": "oper",
+                "expver": "0001",
+            },
+            "tooltip": "Keys in the ecCodes mars namespace",
+        }
+    ]
+    assert r == ref
+
+    # namespace reformatted
+    r = f[0].dump(namespace="mars", _as_raw=False)
+    ref = {
+        "mars": {
+            "domain": "g",
+            "levtype": "pl",
+            "levelist": 1000,
+            "date": 20180801,
+            "time": 1200,
+            "step": 0,
+            "param": "t",
+            "class": "od",
+            "type": "an",
+            "stream": "oper",
+            "expver": "0001",
+        }
+    }
+    assert r == ref
+
+
+if __name__ == "__main__":
+    from earthkit.data.testing import main
+
+    main(__file__)