Repository: NCAS-CMS/cfdm

Issue:
Allow file missing data indicator values to be stored on `NetCDFArray` instances
It can be useful to applications to know the missing data indicator without opening the netCDF file. This is the case for active storage reductions currently being implemented in cf-python.

It is almost always the case that the netCDF file is open and available when the `NetCDFArray` instance is created, so it is easy for these to be stored on the object if desired, e.g. 

```python
>>> n = NetCDFArray(
...     'file.nc', 'ncvar', missing_values={"missing_value": -999.0, "valid_range": (-111, 222)}
... )
```

Discussion:

Patch:
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index ae9504987..576c555fc 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -68,7 +68,7 @@ repos:
   # compatible with 'black' with the lines set to ensure so in the repo's
   # pyproject.toml. Other than that and the below, no extra config is required.
   - repo: https://github.com/pycqa/isort
-    rev: 5.8.0
+    rev: 5.12.0
     hooks:
       - id: isort
         name: isort (python)
diff --git a/Changelog.rst b/Changelog.rst
index 9ca2645bb..7951b55e5 100644
--- a/Changelog.rst
+++ b/Changelog.rst
@@ -7,6 +7,8 @@ Version 1.10.0.3
   (https://github.com/NCAS-CMS/cfdm/issues/241)
 * New keyword parameter to `cfdm.unique_constructs`:
   ``ignore_properties`` (https://github.com/NCAS-CMS/cfdm/issues/240)
+* New keyword parameter to `cfdm.NetCDFArray`: ``missing_values``
+  (https://github.com/NCAS-CMS/cfdm/issues/246)
 * Fixed bug that caused `cf.write` to erroneously change external
   netCDF variable names (https://github.com/NCAS-CMS/cfdm/issues/244)
 
diff --git a/cfdm/cfdmimplementation.py b/cfdm/cfdmimplementation.py
index 497297061..5ee5f2d2e 100644
--- a/cfdm/cfdmimplementation.py
+++ b/cfdm/cfdmimplementation.py
@@ -2168,6 +2168,7 @@ class CFDMImplementation(Implementation):
         mask=True,
         units=False,
         calendar=None,
+        missing_values=None,
     ):
         """Return a netCDF array instance.
 
@@ -2203,9 +2204,15 @@ class CFDMImplementation(Implementation):
 
                 .. versionadded:: (cfdm) 1.10.0.2
 
+            missing_values: `dict`, optional
+                The missing value indicators defined by the netCDF
+                variable attributes.
+
+                .. versionadded:: (cfdm) 1.10.0.3
+
         :Returns:
 
-            NetCDF array instance
+            `NetCDFArray`
 
         """
         cls = self.get_class("NetCDFArray")
@@ -2220,6 +2227,7 @@ class CFDMImplementation(Implementation):
             mask=mask,
             units=units,
             calendar=calendar,
+            missing_values=missing_values,
         )
 
     def initialise_NodeCountProperties(self):
diff --git a/cfdm/data/netcdfarray.py b/cfdm/data/netcdfarray.py
index 264b579f3..01d2123b6 100644
--- a/cfdm/data/netcdfarray.py
+++ b/cfdm/data/netcdfarray.py
@@ -25,6 +25,7 @@ class NetCDFArray(abstract.Array):
         mask=True,
         units=False,
         calendar=False,
+        missing_values=None,
         source=None,
         copy=True,
     ):
@@ -107,6 +108,13 @@ class NetCDFArray(abstract.Array):
 
                 .. versionadded:: (cfdm) 1.10.0.1
 
+            missing_values: `dict`, optional
+                The missing value indicators defined by the netCDF
+                variable attributes. See `get_missing_values` for
+                details.
+
+                .. versionadded:: (cfdm) 1.10.0.3
+
             {{init source: optional}}
 
                 .. versionadded:: (cfdm) 1.10.0.0
@@ -164,6 +172,11 @@ class NetCDFArray(abstract.Array):
             except AttributeError:
                 calendar = False
 
+            try:
+                missing_values = source._get_component("missing_values", None)
+            except AttributeError:
+                missing_values = None
+
         if shape is not None:
             self._set_component("shape", shape, copy=False)
 
@@ -176,6 +189,11 @@ class NetCDFArray(abstract.Array):
         if varid is not None:
             self._set_component("varid", varid, copy=False)
 
+        if missing_values is not None:
+            self._set_component(
+                "missing_values", missing_values.copy(), copy=False
+            )
+
         self._set_component("group", group, copy=False)
         self._set_component("dtype", dtype, copy=False)
         self._set_component("mask", mask, copy=False)
@@ -454,6 +472,41 @@ class NetCDFArray(abstract.Array):
         """
         return self._get_component("mask")
 
+    def get_missing_values(self):
+        """The missing value indicators from the netCDF variable.
+
+        .. versionadded:: (cfdm) 1.10.0.3
+
+        :Returns:
+
+            `dict` or `None`
+                The missing value indicators from the netCDF variable,
+                keyed by their netCDF attribute names. An empty
+                dictionary signifies that no missing values are given
+                in the file. `None` signifies that the missing values
+                have not been set.
+
+        **Examples**
+
+        >>> a.get_missing_values()
+        None
+
+        >>> b.get_missing_values()
+        {}
+
+        >>> c.get_missing_values()
+        {'missing_value': 1e20, 'valid_range': (-10, 20)}
+
+        >>> d.get_missing_values()
+        {'valid_min': -999}
+
+        """
+        out = self._get_component("missing_values", None)
+        if out is None:
+            return
+
+        return out.copy()
+
     def get_ncvar(self):
         """The name of the netCDF variable containing the array.
 
diff --git a/cfdm/read_write/netcdf/netcdfread.py b/cfdm/read_write/netcdf/netcdfread.py
index 29aa0ef19..cbe0c1703 100644
--- a/cfdm/read_write/netcdf/netcdfread.py
+++ b/cfdm/read_write/netcdf/netcdfread.py
@@ -5378,6 +5378,26 @@ class NetCDFRead(IORead):
             # TODO: think using e.g. '/forecasts/model1' has the value for
             # nc_set_variable. What about nc_set_dimension?
 
+        # Store the missing value indicators
+        missing_values = {}
+        for attr in (
+            "missing_value",
+            "_FillValue",
+            "valid_min",
+            "valid_max",
+            "valid_range",
+        ):
+            value = getattr(variable, attr, None)
+            if value is not None:
+                missing_values[attr] = value
+
+        valid_range = missing_values.get("valid_range")
+        if valid_range is not None:
+            try:
+                missing_values["valid_range"] = tuple(valid_range)
+            except TypeError:
+                pass
+
         kwargs = {
             "filename": filename,
             "shape": shape,
@@ -5387,16 +5407,13 @@ class NetCDFRead(IORead):
             "group": group,
             "units": units,
             "calendar": calendar,
+            "missing_values": missing_values,
         }
 
         if return_kwargs_only:
             return kwargs
 
-        array = self.implementation.initialise_NetCDFArray(
-            ndim=ndim,  # TODO: Can we get rid of this?
-            size=size,  # TODO: Can we get rid of this?
-            **kwargs,
-        )
+        array = self.implementation.initialise_NetCDFArray(**kwargs)
 
         return array, kwargs
 


Test patch:
diff --git a/cfdm/test/test_NetCDFArray.py b/cfdm/test/test_NetCDFArray.py
new file mode 100644
index 000000000..8bd75d697
--- /dev/null
+++ b/cfdm/test/test_NetCDFArray.py
@@ -0,0 +1,78 @@
+import atexit
+import datetime
+import faulthandler
+import os
+import tempfile
+import unittest
+
+faulthandler.enable()  # to debug seg faults and timeouts
+
+import cfdm
+
+n_tmpfiles = 1
+tmpfiles = [
+    tempfile.mkstemp("_test_netCDF.nc", dir=os.getcwd())[1]
+    for i in range(n_tmpfiles)
+]
+(tmpfile,) = tmpfiles
+
+
+def _remove_tmpfiles():
+    """Remove temporary files created during tests."""
+    for f in tmpfiles:
+        try:
+            os.remove(f)
+        except OSError:
+            pass
+
+
+atexit.register(_remove_tmpfiles)
+
+
+class NetCDFTest(unittest.TestCase):
+    """Unit test for the NetCDF class."""
+
+    def setUp(self):
+        """Preparations called immediately before each test method."""
+        # Disable log messages to silence expected warnings
+        cfdm.log_level("DISABLE")
+        # Note: to enable all messages for given methods, lines or
+        # calls (those without a 'verbose' option to do the same)
+        # e.g. to debug them, wrap them (for methods, start-to-end
+        # internally) as follows:
+        #
+        # cfdm.log_level('DEBUG')
+        # < ... test code ... >
+        # cfdm.log_level('DISABLE')
+
+    def test_NetCDFArray_get_missing_values(self):
+        """Test NetCDFArray.get_missing_values."""
+        f = cfdm.example_field(0)
+
+        f.set_property("missing_value", -999)
+        f.set_property("_FillValue", -3)
+        f.set_property("valid_range", [-111, 222])
+        cfdm.write(f, tmpfile)
+
+        g = cfdm.read(tmpfile)[0]
+        self.assertEqual(
+            g.data.source().get_missing_values(),
+            {
+                "missing_value": -999.0,
+                "_FillValue": -3,
+                "valid_range": (-111, 222),
+            },
+        )
+
+        c = g.coordinate("latitude")
+        self.assertEqual(c.data.source().get_missing_values(), {})
+
+        a = cfdm.NetCDFArray("file.nc", "ncvar")
+        self.assertIsNone(a.get_missing_values())
+
+
+if __name__ == "__main__":
+    print("Run date:", datetime.datetime.now())
+    cfdm.environment()
+    print("")
+    unittest.main(verbosity=2)