Repository: pydap/pydap
Pull request #453 (2025-03-14 22:52:04)

Collect/provide more metadata about dimension names per variable
<!-- Thank you very much for your hard work and contribution! -->

The DRAFT following Pull Request:

- [x] Closes an #452 with feature about returning dimension names (per variable) as a result of method `ds.variables()`
- [x] fixes latest bug in #450 
- [x] Tests are added to demonstrate the new or fixed behavior, and all tests pass
on a local environment.

This PR enables a mix of fully-qualifying and non-fully qualifying names when dimensions of variables spans several groups.

### example 
```python
ds = open_url("dap4://test.opendap.org:8080/opendap/dap4/SimpleGroup.nc4.h5")
ds.tree()
>>>
.SimpleGroup.nc4.h5
├──SimpleGroup
│  ├──Y
│  ├──X
│  ├──Temperature
│  └──Salinity
├──time
├──Z
├──Pressure
└──time_bnds
```

1) At the root level identify variables `time, Z, Pressure, time_bnds` and their dimensions `dims` (relative names). If each dim in `dims` is defined at same level (hierarcy) these are non-qualifying names.
```python
{'time': {'dtype': dtype('>f4'), 'shape': (1,), 'dims': ['time']},
 'Z': {'dtype': dtype('>f4'), 'shape': (1000,), 'dims': ['Z']},
 'Pressure': {'dtype': dtype('>f4'), 'shape': (1000,), 'dims': ['Z']},
 'time_bnds': {'dtype': dtype('>f4'), 'shape': (1, 2), 'dims': ['time', 'nv']}}
```
2) Within `SimpleGroup`, similarly as before, identifies `X, Y, Temperature, Salinity` and their dims. If each dim is local within `SimpleGroup`, its name does not contain slashes `/` (non-qualifying). In the case of dimension `time`, it is defined at root level (not current hierarchy) and so its name is a fully qualifying name.=

```python
{'Y': {'dtype': dtype('>i2'), 'shape': (40,), 'dims': ['Y']},
 'X': {'dtype': dtype('>i2'), 'shape': (40,), 'dims': ['X']},
 'Temperature': {'dtype': dtype('>f4'),
  'shape': (1, 40, 40),
  'dims': ['/time', 'Y', 'X']},
 'Salinity': {'dtype': dtype('>f4'),
  'shape': (1, 40, 40),
  'dims': ['/time', 'Y', 'X']}}
```


Patch:
diff --git a/src/pydap/handlers/dap.py b/src/pydap/handlers/dap.py
index 29986d8..94e81f7 100644
--- a/src/pydap/handlers/dap.py
+++ b/src/pydap/handlers/dap.py
@@ -92,6 +92,11 @@ class DAPHandler(BaseHandler):
             if protocol not in ["dap2", "dap4"]:
                 raise TypeError("protocol must be one of `dap2` or `dap4")
             self.protocol = protocol
+            if self.scheme == protocol:
+                # the other alternative occurs during testing
+                # the server - only when protocol and scheme match,
+                # should pydap change the scheme provided by user
+                self.scheme = "http"
         else:
             self.protocol = self.determine_protocol()
         self.get_kwargs = get_kwargs or {}
diff --git a/src/pydap/model.py b/src/pydap/model.py
index f89702e..f5764c5 100644
--- a/src/pydap/model.py
+++ b/src/pydap/model.py
@@ -261,7 +261,6 @@ class BaseType(DapType):
         super(BaseType, self).__init__(name, attributes, **kwargs)
         self.data = data
         self.dimensions = dimensions or ()
-
         # these are set when not data is present (eg, when parsing a DDS)
         self._dtype = None
         self._shape = ()
@@ -273,7 +272,10 @@ class BaseType(DapType):
 
     @property
     def path(self):
-        return self.data.path
+        try:
+            return self.data.path
+        except AttributeError:
+            return None
 
     @property
     def dtype(self):
@@ -283,7 +285,10 @@ class BaseType(DapType):
     @property
     def shape(self):
         """Property that returns the data shape."""
-        return self.data.shape
+        try:
+            return self.data.shape
+        except AttributeError:
+            return self._shape
 
     def reshape(self, *args):
         """Method that reshapes the data:"""
@@ -526,7 +531,7 @@ class StructureType(DapType, Mapping):
     def type(self):
         return "Structure"
 
-    def structures(self):
+    def structures(self) -> dict:
         out = {}
         Gs = [key for key in self.children() if isinstance(key, StructureType)]
         Strs = [key for key in Gs if key.type == "Structure"]
@@ -534,7 +539,7 @@ class StructureType(DapType, Mapping):
             out.update({var.name: [key.name for key in var.children()]})
         return out
 
-    def groups(self):
+    def groups(self) -> dict:
         """Returns fqn for all (nested) groups"""
         out = {}
         for var in walk(self, GroupType):
@@ -542,24 +547,33 @@ class StructureType(DapType, Mapping):
                 out.update({var.name: var.path})
         return out
 
-    def sequences(self):
+    def sequences(self) -> dict:
+        "returns all (nested) sequences"
         out = {}
         for var in walk(self, SequenceType):
             if var.type == "Sequence":
                 out.update({var.name: [key.name for key in var.children()]})
         return out
 
-    def grids(self):
+    def grids(self) -> dict:
+        """returns all GridType (DAP2) objects in a"""
         out = {}
         for var in walk(self, GridType):
             out.update({var.name: {"shape": var.shape, "maps": list(var.maps)}})
         return out
 
-    def variables(self):
+    def variables(self) -> dict:
+        """returns all variables at the present hierarcy"""
         out = {}
         Bcs = [key for key in self.children() if isinstance(key, BaseType)]
         for var in Bcs:
-            out.update({var.name: var.dtype})
+            if "dims" in var.attributes:
+                dims = var.dims
+            else:
+                dims = []
+            out.update(
+                {var.name: {"dtype": var.dtype, "shape": var.shape, "dims": dims}}
+            )
         return out
 
 
diff --git a/src/pydap/parsers/dmr.py b/src/pydap/parsers/dmr.py
index 2194408..c068109 100644
--- a/src/pydap/parsers/dmr.py
+++ b/src/pydap/parsers/dmr.py
@@ -286,20 +286,33 @@ def dmr_to_dataset(dmr):
             var_name = parts[-1]
             path = ("/").join(parts[:-1])
             variable["attributes"]["path"] = path
-
         data = DummyData(dtype=variable["dtype"], shape=variable["shape"], path=path)
         # make sure all dimensions have qualifying name
-        Dims = []
+        Dims = []  # dims with fully qualifying names (fqn)
+        nqfDims = []  # non-fqn when `dims` and `vars` share same path
         for dim in variable["dims"]:
-            if len(dim.split(split_by)) == 1:
+            parts = dim.split(split_by)
+            if len(parts) == 1:
                 Dims.append("/" + dim)
-            else:
+            else:  # there is a group
                 Dims.append(dim)
+            if len(parts) == len(variable["name"].split(split_by)):
+                # if path to dim is identical to path to variable
+                dim_name = parts[-1]  # only keep the local dim name
+                nqfDims.append(dim_name)
+            else:
+                # dimension name will have a fully qualifying name
+                if len(parts) == 1:
+                    nqfDims.append("/" + dim)
+                else:
+                    nqfDims.append(dim)
+
         # pass along maps
         var_kwargs = {
             "name": pydap.lib._quote(name),
             "data": data,
             "dimensions": Dims,
+            "dims": nqfDims,
             "attributes": variable["attributes"],
         }
         if "maps" in variable.keys():


Test patch:
diff --git a/src/pydap/tests/test_handlers_dap.py b/src/pydap/tests/test_handlers_dap.py
index 3523c0b..0069bfc 100644
--- a/src/pydap/tests/test_handlers_dap.py
+++ b/src/pydap/tests/test_handlers_dap.py
@@ -5,6 +5,7 @@ import unittest
 
 import numpy as np
 import pytest
+from requests.utils import urlparse
 from webob.response import Response
 
 import pydap.model
@@ -357,6 +358,9 @@ def test_protocols(url, app, expect):
                 DAPHandler(url=url)
         else:
             assert DAPHandler(url=url).protocol == expect
+    if urlparse(url).scheme in ["dap2", "dap4"]:
+        assert DAPHandler(url=url).protocol == expect
+        assert isinstance(DAPHandler(url=url).dataset, DatasetType)
     with pytest.raises(TypeError):
         # only protocol='dap2' or `dap4` is supported
         DAPHandler(url=url, application=app, protocol="errdap")
@@ -660,7 +664,10 @@ class TestUnpackDap4Data(unittest.TestCase):
 
     def testDMR(self):
         ds = dmr_to_dataset(self.unpacker.dmr)
-        self.assertEqual(ds.variables(), {"SST": np.dtype(">f4")})
+        self.assertEqual(
+            ds.variables(),
+            {"SST": {"dtype": np.dtype(">f4"), "shape": (1, 4, 4), "dims": []}},
+        )
 
     def testResponse(self):
         self.assertIsInstance(self.unpacker.r, Response)
diff --git a/src/pydap/tests/test_model.py b/src/pydap/tests/test_model.py
index 32c79fd..99c534f 100644
--- a/src/pydap/tests/test_model.py
+++ b/src/pydap/tests/test_model.py
@@ -541,8 +541,12 @@ def test_DatasetType_variables():
     dataset.createGroup("/Group1")
     dataset.createVariable("/root_variable", dtype=np.float32)
     dataset.createVariable("/Group1/other_variable", dtype=np.int64)
-    assert dataset.variables() == {"root_variable": np.dtype("float32")}
-    assert dataset["/Group1"].variables() == {"other_variable": np.dtype("int64")}
+    assert dataset.variables() == {
+        "root_variable": {"dtype": np.dtype("float32"), "shape": (), "dims": []}
+    }
+    assert dataset["/Group1"].variables() == {
+        "other_variable": {"dtype": np.dtype("int64"), "shape": (), "dims": []}
+    }
 
 
 # Test pydap grids.
diff --git a/src/pydap/tests/test_parsers_dmr.py b/src/pydap/tests/test_parsers_dmr.py
index 9f8840d..a98a74f 100644
--- a/src/pydap/tests/test_parsers_dmr.py
+++ b/src/pydap/tests/test_parsers_dmr.py
@@ -122,6 +122,17 @@ class TestDMRParser(unittest.TestCase):
         # assert nv is NOT a variable/array
         self.assertNotIn("nv", variables)
 
+    def test_non_fqn_dims(self):
+        """Test that highlights the different behavior between variable attribtues
+        `dims` (semi-fully qualifgying names) and `dimensions` (fully-qualifying name)
+        """
+        dataset = load_dmr_file("data/dmrs/SimpleGroup.dmr")
+        self.assertEqual(dataset["/SimpleGroup/Temperature"].dims, ["/time", "Y", "X"])
+        self.assertEqual(
+            dataset["/SimpleGroup/Temperature"].dimensions,
+            ["/time", "/SimpleGroup/Y", "/SimpleGroup/X"],
+        )
+
     def tests_FlatGroups(self):
         dataset = load_dmr_file("data/dmrs/SimpleGroupFlat.dmr")
         # pick a single variable Maps