Repository: pydap/pydap
Pull request #345 (2024-08-08 22:49:36)

Parse NaN attribute values on DMR (DAP4)
<!-- Thank you very much for your hard work and contribution! -->

The following Pull Request:

- [x] Closes issue #344 . More description there!
- [x] Tests are added to demonstrate the new or fixed behavior, and all tests pass
on a local environment.




Patch:
diff --git a/src/pydap/parsers/dmr.py b/src/pydap/parsers/dmr.py
index 7a12e11..98316e2 100644
--- a/src/pydap/parsers/dmr.py
+++ b/src/pydap/parsers/dmr.py
@@ -82,13 +82,38 @@ def get_dtype(element):
 
 def get_attributes(element, attributes={}):
     attribute_elements = element.findall("Attribute")
-    numType = [item for item in dmr_atomic_types if item not in ["Byte", "Char"]]
+    Float_types = ["Float32", "float", "Float64"]
+    Int_types = ["Int16", "Int32", "Int64", "int", "Int8"]
+    uInt_types = ["uInt16", "uInt32", "uInt64", "uint", "uInt8"]
     for attribute_element in attribute_elements:
         name = attribute_element.get("name")
         value = attribute_element.find("Value").text
         _type = attribute_element.get("type")
-        if _type in numType:
-            value = ast.literal_eval(value)
+        if _type in dmr_atomic_types:
+            if _type in Float_types:
+                # keep float-type of value
+                value = float(value)
+            elif _type in Int_types or _type in uInt_types:
+                # keeps integer-type of value
+                value = int(value)
+            else:
+                try:
+                    value = ast.literal_eval(value)
+                except ValueError:
+                    # leaves value as string
+                    raise Warning(
+                        """
+                        Pydap could not turn the non-string Attribute: `{}`
+                        with value `{}` into a numeric type during parsing
+                        of the `DMR`. Parsing as string value instead. If you
+                        think this is a bug consider raising a issue. To
+                        learn more about DAP's atomic types, go to:
+                        https://docs.opendap.org/index.php/
+                        DAP4:_Specification_Volume_1#Atomic_Types
+                        """.format(
+                            name, value
+                        )
+                    )
         attributes[name] = value
     return attributes
 


Test patch:
diff --git a/src/pydap/tests/data/dmrs/SimpleGroup.dmr b/src/pydap/tests/data/dmrs/SimpleGroup.dmr
index 24785ad..394ef31 100644
--- a/src/pydap/tests/data/dmrs/SimpleGroup.dmr
+++ b/src/pydap/tests/data/dmrs/SimpleGroup.dmr
@@ -9,6 +9,9 @@
             <Attribute name="units" type="str">
                 <Value>degrees_celsius</Value>
             </Attribute>
+            <Attribute name="_FillValue" type="float">
+                <Value>inf</Value>
+            </Attribute>
         </Float32>
         <Float32 name="Salinity">
             <Dim name="/SimpleGroup/X"/>
@@ -16,6 +19,9 @@
             <Attribute name="units" type="str">
                 <Value>psu</Value>
             </Attribute>
+            <Attribute name="_FillValue" type="float">
+                <Value>nan</Value>
+            </Attribute>
         </Float32>
         <Int16 name="X">
             <Dim name="/SimpleGroup/X"/>
diff --git a/src/pydap/tests/datasets.py b/src/pydap/tests/datasets.py
index 59a74d1..8c64738 100644
--- a/src/pydap/tests/datasets.py
+++ b/src/pydap/tests/datasets.py
@@ -231,12 +231,14 @@ SimpleGroup["SimpleGroup"]["Temperature"] = BaseType(
     np.arange(10, 26, 1, dtype="f4").reshape(4, 4),
     units="degrees_celsius",
     dimensions=("X", "Y"),
+    _FillValue=np.inf,
 )
 SimpleGroup["SimpleGroup"]["Salinity"] = BaseType(
     "Salinity",
     30 * np.ones(16, dtype="f4").reshape(4, 4),
     units="psu",
     dimensions=("X", "Y"),
+    _FillValue=np.nan,
 )
 SimpleGroup["/SimpleGroup/X"] = BaseType("X", np.arange(4, dtype="i2"), dimensions="X")
 SimpleGroup["/SimpleGroup/Y"] = BaseType("Y", np.arange(4, dtype="i2"), dimensions="Y")
diff --git a/src/pydap/tests/test_parsers_dmr.py b/src/pydap/tests/test_parsers_dmr.py
index 984bbc8..9b06932 100644
--- a/src/pydap/tests/test_parsers_dmr.py
+++ b/src/pydap/tests/test_parsers_dmr.py
@@ -1,64 +1,97 @@
 """Test DAP4 parsing functions."""
 
 import os
+import unittest
 
-import pydap.parsers.dmr
+import numpy as np
+
+from ..parsers.dmr import dmr_to_dataset
 
 
 def load_dmr_file(file_path):
     abs_path = os.path.join(os.path.dirname(__file__), file_path)
     with open(abs_path, "r") as dmr_file:
         dmr = dmr_file.read()
-    dataset = pydap.parsers.dmr.dmr_to_dataset(dmr)
+    dataset = dmr_to_dataset(dmr)
     return dataset
 
 
-def test_single_scalar():
-    """Test a single scalar case."""
-    single_scalar_dmr = """<Dataset name="foo"><Int32 name="x"/></Dataset>"""
-    dataset = pydap.parsers.dmr.dmr_to_dataset(single_scalar_dmr)
-    assert dataset["x"].dtype == ">i4"
-
-
-def test_coads_climatology2():
-    dataset = load_dmr_file("data/dmrs/coads_climatology.nc.dmr")
-    assert dataset["SST"].attributes["long_name"] == "SEA SURFACE TEMPERATURE"
-    assert dataset["SST"].attributes["missing_value"] == -9.99999979e33
-    assert dataset["AIRT"].shape == (12, 90, 180)
-    assert dataset["VWND"].dtype.str == ">f4"
-
-
-def test_atl03():
-    # Contains groups
-    dataset = load_dmr_file(
-        "data/dmrs/ATL03_20181228015957_13810110_003_01.2var.h5.dmrpp.dmr"
-    )
-    assert (
-        dataset["/gt1r/bckgrd_atlas/bckgrd_int_height"].attributes["contentType"]
-        == "modelResult"
-    )
-
-
-def test_jpl():
-    # Contains groups
-    dataset = load_dmr_file(
-        "data/dmrs/20220102090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.dmr"
-    )
-    assert dataset["sea_ice_fraction"].dimensions == ["time", "lat", "lon"]
-
-
-def test_mod05():
-    dataset = load_dmr_file(
-        "data/dmrs/MOD05_L2.A2019336.2315.061.2019337071952.hdf.dmr"
-    )
-    assert dataset["Water_Vapor_Infrared"].dimensions == [
-        "Cell_Along_Swath_5km",
-        "Cell_Across_Swath_5km",
-    ]
-
-
-def test_SWOT():
-    dataset = load_dmr_file("data/dmrs/SWOT_GPR.dmr")
-    # pick a single variable Maps
-    maps = ("/data_01/longitude", "/data_01/latitude")
-    assert dataset["data_01/ku/swh_ocean"].Maps == maps
+class DMRParser(unittest.TestCase):
+    """Test parsing a DMR."""
+
+    def test_single_scalar(self):
+        """Test a single scalar case."""
+        single_scalar_dmr = """<Dataset name="foo"><Int32 name="x"/></Dataset>"""
+        dataset = dmr_to_dataset(single_scalar_dmr)
+        self.assertEqual(dataset["x"].dtype, ">i4")
+
+    def test_missing_value(self):
+        """Test cases with missing value."""
+        NAN_dmr = """<Dataset name="foo">\n    <Int32 name="x">\n
+                <Attribute name="missing_value" type="Float32">\n
+                            <Value>NaN</Value>\n        </Attribute>\n
+                                </Int32>\n</Dataset>"""
+        INF_dmr = """<Dataset name="foo">\n    <Int32 name="x">\n
+                <Attribute name="missing_value" type="Float32">\n
+                            <Value>Inf</Value>\n        </Attribute>\n
+                                </Int32>\n</Dataset>"""
+
+        dataset_nan = dmr_to_dataset(NAN_dmr)
+        dataset_inf = dmr_to_dataset(INF_dmr)
+
+        X_nan = dataset_nan["x"]
+        X_inf = dataset_inf["x"]
+        self.assertEqual(np.isnan(X_nan.attributes["missing_value"]), True)
+        self.assertEqual(np.isinf(X_inf.attributes["missing_value"]), True)
+
+    def test_attribute_warning(self):
+        byte_dmr = """<Dataset name="foo">\n    <Int32 name="x">\n
+                <Attribute name="missing_value" type="Byte">\n
+                            <Value>x00</Value>\n        </Attribute>\n
+                                </Int32>\n</Dataset>"""
+        with self.assertRaises(Warning):
+            dmr_to_dataset(byte_dmr)
+
+    def test_coads_climatology2(self):
+        dataset = load_dmr_file("data/dmrs/coads_climatology.nc.dmr")
+        self.assertEqual(
+            dataset["SST"].attributes["long_name"], "SEA SURFACE TEMPERATURE"
+        )
+        self.assertEqual(dataset["SST"].attributes["missing_value"], -9.99999979e33)
+        self.assertEqual(dataset["AIRT"].shape, (12, 90, 180))
+        self.assertEqual(dataset["VWND"].dtype.str, ">f4")
+
+    def test_atl03(self):
+        # Contains groups
+        dataset = load_dmr_file(
+            "data/dmrs/ATL03_20181228015957_13810110_003_01.2var.h5.dmrpp.dmr"
+        )
+        self.assertEqual(
+            dataset["/gt1r/bckgrd_atlas/bckgrd_int_height"].attributes["contentType"],
+            "modelResult",
+        )
+
+    def test_jpl(self):
+        # Contains groups
+        dataset = load_dmr_file(
+            "data/dmrs/20220102090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.dmr"
+        )
+        self.assertEqual(dataset["sea_ice_fraction"].dimensions, ["time", "lat", "lon"])
+
+    def test_mod05(self):
+        dataset = load_dmr_file(
+            "data/dmrs/MOD05_L2.A2019336.2315.061.2019337071952.hdf.dmr"
+        )
+        self.assertEqual(
+            dataset["Water_Vapor_Infrared"].dimensions,
+            [
+                "Cell_Along_Swath_5km",
+                "Cell_Across_Swath_5km",
+            ],
+        )
+
+    def test_SWOT(self):
+        dataset = load_dmr_file("data/dmrs/SWOT_GPR.dmr")
+        # pick a single variable Maps
+        maps = ("/data_01/longitude", "/data_01/latitude")
+        self.assertEqual(dataset["data_01/ku/swh_ocean"].Maps, maps)
diff --git a/src/pydap/tests/test_responses_dmr.py b/src/pydap/tests/test_responses_dmr.py
index f059f1c..81f7a51 100644
--- a/src/pydap/tests/test_responses_dmr.py
+++ b/src/pydap/tests/test_responses_dmr.py
@@ -58,7 +58,7 @@ class TestDMRResponseSequence(unittest.TestCase):
                         "Access-Control-Allow-Headers",
                         "Origin, X-Requested-With, Content-Type",
                     ),
-                    ("Content-Length", "989"),
+                    ("Content-Length", "1219"),
                 ]
             ),
         )