Repository: zarr-developers/VirtualiZarr
Pull request #542: Use ManifesStore to_virtual_variable directly in HDFVirtualBackend.
Author: login: sharkinsspatial
Created: 2025-04-10T17:04:10Z | Merged: 2025-04-21T18:41:06Z

<!-- Feel free to remove check-list items aren't relevant to your change -->

- [ ] Closes #xxxx
- [x] Tests added
- [x] Tests passing
- [x] Full type hint coverage
- [ ] Changes are documented in `docs/releases.rst`
- [ ] New functions/methods are listed in `api.rst`
- [ ] New functionality has documentation



Diff:
diff --git a/pyproject.toml b/pyproject.toml
index f90ea87bc..67202cfca 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -50,6 +50,7 @@ hdf = [
     "hdf5plugin",
     "imagecodecs",
     "imagecodecs-numcodecs==2024.6.1",
+    "obstore>=0.5.1",
 ]
 
 # kerchunk-based readers
diff --git a/virtualizarr/manifests/store.py b/virtualizarr/manifests/store.py
index ccc00b690..a9d7e6d94 100644
--- a/virtualizarr/manifests/store.py
+++ b/virtualizarr/manifests/store.py
@@ -150,7 +150,9 @@ def default_object_store(filepath: str) -> ObjectStore:
             virtual_hosted_style_request=False,
             region=_find_bucket_region(bucket),
         )
-
+    if parsed.scheme in ["http", "https"]:
+        base_url = f"{parsed.scheme}://{parsed.netloc}"
+        return obs.store.HTTPStore.from_url(base_url)
     raise NotImplementedError(f"{parsed.scheme} is not yet supported")
 
 
diff --git a/virtualizarr/readers/hdf/hdf.py b/virtualizarr/readers/hdf/hdf.py
index c6947ccd9..37c77a385 100644
--- a/virtualizarr/readers/hdf/hdf.py
+++ b/virtualizarr/readers/hdf/hdf.py
@@ -4,9 +4,6 @@
 from pathlib import Path
 from typing import (
     TYPE_CHECKING,
-    Any,
-    Dict,
-    Hashable,
     Iterable,
     List,
     Mapping,
@@ -31,13 +28,9 @@
 from virtualizarr.manifests.store import ObjectStoreRegistry, default_object_store
 from virtualizarr.manifests.utils import create_v3_array_metadata
 from virtualizarr.readers.api import VirtualBackend
-from virtualizarr.readers.hdf.filters import cfcodec_from_dataset, codecs_from_dataset
+from virtualizarr.readers.hdf.filters import codecs_from_dataset
 from virtualizarr.types import ChunkKey
-from virtualizarr.utils import _FsspecFSFromFilepath, soft_import
-from virtualizarr.xarray import (
-    construct_fully_virtual_dataset,
-    construct_virtual_dataset,
-)
+from virtualizarr.utils import soft_import
 
 h5py = soft_import("h5py", "For reading hdf files", strict=False)
 
@@ -88,6 +81,22 @@ def _construct_manifest_array(
         attrs = HDFVirtualBackend._extract_attrs(dataset)
         dtype = dataset.dtype
 
+        # Temporarily disable use CF->Codecs - TODO re-enable in subsequent PR.
+        # cfcodec = cfcodec_from_dataset(dataset)
+        # if cfcodec:
+        # codecs.insert(0, cfcodec["codec"])
+        # dtype = cfcodec["target_dtype"]
+        # attrs.pop("scale_factor", None)
+        # attrs.pop("add_offset", None)
+        # else:
+        # dtype = dataset.dtype
+
+        if "_FillValue" in attrs:
+            encoded_cf_fill_value = HDFVirtualBackend._encode_cf_fill_value(
+                attrs["_FillValue"], dtype
+            )
+            attrs["_FillValue"] = encoded_cf_fill_value
+
         codec_configs = [
             numcodec_config_to_configurable(codec.get_config()) for codec in codecs
         ]
@@ -113,7 +122,7 @@ def _construct_manifest_group(
         filepath: str,
         *,
         group: str | None = None,
-        drop_variables: Optional[List[str]] = None,
+        drop_variables: Optional[Iterable[str]] = None,
     ) -> ManifestGroup:
         """
         Construct a virtual Group from a HDF dataset.
@@ -136,11 +145,16 @@ def _construct_manifest_group(
             group_name = "/"
 
         manifest_dict = {}
-        non_coordinate_dimesion_vars = HDFVirtualBackend._find_non_coord_dimension_vars(
-            group=g
+        # Several of our test fixtures which use xr.tutorial data have
+        # non coord dimensions serialized using big endian dtypes which are not
+        # yet supported in zarr-python v3.  We'll drop these variables for the
+        # moment until big endian support is included upstream.)
+
+        non_coordinate_dimension_vars = (
+            HDFVirtualBackend._find_non_coord_dimension_vars(group=g)
         )
-        drop_variables = list(set(drop_variables + non_coordinate_dimesion_vars))
-        attrs: dict[str, Any] = {}
+        drop_variables = list(set(list(drop_variables) + non_coordinate_dimension_vars))
+        attrs = HDFVirtualBackend._extract_attrs(g)
         for key in g.keys():
             if key not in drop_variables:
                 if isinstance(g[key], h5py.Dataset):
@@ -159,12 +173,16 @@ def _create_manifest_store(
         *,
         store: ObjectStore | None = None,
         group: str | None = None,
+        drop_variables: Iterable[str] | None = None,
     ) -> ManifestStore:
         # Create a group containing dataset level metadata and all the manifest arrays
         if not store:
             store = default_object_store(filepath)  # type: ignore
         manifest_group = HDFVirtualBackend._construct_manifest_group(
-            store=store, filepath=filepath, group=group
+            store=store,
+            filepath=filepath,
+            group=group,
+            drop_variables=drop_variables,
         )
         registry = ObjectStoreRegistry({filepath: store})
         # Convert to a manifest store
@@ -192,40 +210,21 @@ def open_virtual_dataset(
             filepath, fs_root=Path.cwd().as_uri()
         )
 
-        _drop_vars: list[Hashable] = (
+        _drop_vars: Iterable[str] = (
             [] if drop_variables is None else list(drop_variables)
         )
 
-        # TODO provide a way to drop a variable _before_ h5py attempts to inspect it?
-        virtual_vars = HDFVirtualBackend._virtual_vars_from_hdf(
-            path=filepath,
-            group=group,
-            reader_options=reader_options,
-        )
-
-        attrs = HDFVirtualBackend._get_group_attrs(
-            path=filepath, reader_options=reader_options, group=group
-        )
-        coordinates_attr = attrs.pop("coordinates", "")
-        coord_names = coordinates_attr.split()
-
-        fully_virtual_dataset = construct_fully_virtual_dataset(
-            virtual_vars=virtual_vars,
-            coord_names=coord_names,
-            attrs=attrs,
-        )
-
-        vds = construct_virtual_dataset(
-            fully_virtual_ds=fully_virtual_dataset,
+        manifest_store = HDFVirtualBackend._create_manifest_store(
             filepath=filepath,
+            drop_variables=_drop_vars,
             group=group,
+        )
+        ds = manifest_store.to_virtual_dataset(
             loadable_variables=loadable_variables,
-            reader_options=reader_options,
-            indexes=indexes,
             decode_times=decode_times,
+            indexes=indexes,
         )
-
-        return vds.drop_vars(_drop_vars)
+        return ds
 
     @staticmethod
     def _dataset_chunk_manifest(
@@ -346,29 +345,6 @@ def _dataset_dims(dataset: H5Dataset, group: str = "") -> List[str]:
 
         return [dim.removeprefix(group) for dim in dims]
 
-    @staticmethod
-    def _extract_cf_fill_value(
-        h5obj: Union[H5Dataset, H5Group],
-    ) -> Optional[FillValueType]:
-        """
-        Convert the _FillValue attribute from an HDF5 group or dataset into
-        encoding.
-
-        Parameters
-        ----------
-        h5obj : h5py.Group or h5py.Dataset
-            An h5py group or dataset.
-        """
-        fillvalue = None
-        for n, v in h5obj.attrs.items():
-            if n == "_FillValue":
-                if isinstance(v, np.ndarray) and v.size == 1:
-                    fillvalue = v.item()
-                else:
-                    fillvalue = v
-                fillvalue = FillValueCoder.encode(fillvalue, h5obj.dtype)  # type: ignore[arg-type]
-        return fillvalue
-
     @staticmethod
     def _extract_attrs(h5obj: Union[H5Dataset, H5Group]):
         """
@@ -394,7 +370,7 @@ def _extract_attrs(h5obj: Union[H5Dataset, H5Group]):
             if n in _HIDDEN_ATTRS:
                 continue
             if n == "_FillValue":
-                continue
+                v = v
             # Fix some attribute values to avoid JSON encoding exceptions...
             if isinstance(v, bytes):
                 v = v.decode("utf-8") or " "
@@ -414,148 +390,6 @@ def _extract_attrs(h5obj: Union[H5Dataset, H5Group]):
             attrs[n] = v
         return attrs
 
-    @staticmethod
-    def _dataset_to_variable(
-        path: str,
-        dataset: H5Dataset,
-        group: str,
-    ) -> Optional[xr.Variable]:
-        """
-        Extract an xarray Variable with ManifestArray data from an h5py dataset
-
-        Parameters
-        ----------
-        dataset : h5py.Dataset
-            An h5py dataset.
-        group : str
-            Name of the group containing this h5py.Dataset.
-
-        Returns
-        -------
-        list: xarray.Variable
-            A list of xarray variables.
-        """
-        chunks = dataset.chunks if dataset.chunks else dataset.shape
-        codecs = codecs_from_dataset(dataset)
-        cfcodec = cfcodec_from_dataset(dataset)
-        attrs = HDFVirtualBackend._extract_attrs(dataset)
-        cf_fill_value = HDFVirtualBackend._extract_cf_fill_value(dataset)
-        attrs.pop("_FillValue", None)
-
-        if cfcodec:
-            codecs.insert(0, cfcodec["codec"])
-            dtype = cfcodec["target_dtype"]
-            attrs.pop("scale_factor", None)
-            attrs.pop("add_offset", None)
-        else:
-            dtype = dataset.dtype
-
-        codec_configs = [
-            numcodec_config_to_configurable(codec.get_config()) for codec in codecs
-        ]
-
-        fill_value = dataset.fillvalue.item()
-        metadata = create_v3_array_metadata(
-            shape=dataset.shape,
-            data_type=dtype,
-            chunk_shape=chunks,
-            fill_value=fill_value,
-            codecs=codec_configs,
-        )
-        dims = HDFVirtualBackend._dataset_dims(dataset, group=group)
-        manifest = HDFVirtualBackend._dataset_chunk_manifest(path, dataset)
-        if manifest:
-            marray = ManifestArray(metadata=metadata, chunkmanifest=manifest)
-            variable = xr.Variable(data=marray, dims=dims, attrs=attrs)
-        else:
-            variable = xr.Variable(data=np.empty(dataset.shape), dims=dims, attrs=attrs)
-        if cf_fill_value is not None:
-            variable.encoding["_FillValue"] = cf_fill_value
-        return variable
-
-    @staticmethod
-    def _virtual_vars_from_hdf(
-        path: str,
-        group: Optional[str] = None,
-        drop_variables: Optional[List[str]] = None,
-        reader_options: Optional[dict] = {
-            "storage_options": {"key": "", "secret": "", "anon": True}
-        },
-    ) -> Dict[str, xr.Variable]:
-        """
-        Extract xarray Variables with ManifestArray data from an HDF file or group
-
-        Parameters
-        ----------
-        path: str
-            The path of the hdf5 file.
-        group: str, optional
-            The name of the group for which to extract variables. None refers to the root group.
-        drop_variables: list of str
-            A list of variable names to skip extracting.
-        reader_options: dict
-            A dictionary of reader options passed to fsspec when opening the file.
-
-        Returns
-        -------
-        dict
-            A dictionary of Xarray Variables with the variable names as keys.
-        """
-        if drop_variables is None:
-            drop_variables = []
-
-        open_file = _FsspecFSFromFilepath(
-            filepath=path, reader_options=reader_options
-        ).open_file()
-        f = h5py.File(open_file, mode="r")
-
-        if group is not None and group != "":
-            g = f[group]
-            group_name = group
-            if not isinstance(g, h5py.Group):
-                raise ValueError("The provided group is not an HDF group")
-        else:
-            g = f["/"]
-            group_name = "/"
-
-        variables = {}
-        non_coordinate_dimesion_vars = HDFVirtualBackend._find_non_coord_dimension_vars(
-            group=g
-        )
-        drop_variables = list(set(drop_variables + non_coordinate_dimesion_vars))
-        for key in g.keys():
-            if key not in drop_variables:
-                if isinstance(g[key], h5py.Dataset):
-                    variable = HDFVirtualBackend._dataset_to_variable(
-                        path=path,
-                        dataset=g[key],
-                        group=group_name,
-                    )
-                    if variable is not None:
-                        variables[key] = variable
-        return variables
-
-    @staticmethod
-    def _get_group_attrs(
-        path: str,
-        group: Optional[str] = None,
-        reader_options: Optional[dict] = {
-            "storage_options": {"key": "", "secret": "", "anon": True}
-        },
-    ):
-        open_file = _FsspecFSFromFilepath(
-            filepath=path, reader_options=reader_options
-        ).open_file()
-        f = h5py.File(open_file, mode="r")
-        if group:
-            g = f[group]
-            if not isinstance(g, h5py.Group):
-                raise ValueError("The provided group is not an HDF group")
-        else:
-            g = f
-        attrs = HDFVirtualBackend._extract_attrs(g)
-        return attrs
-
     @staticmethod
     def _find_non_coord_dimension_vars(group: H5Group) -> List[str]:
         dimension_names = []
@@ -569,3 +403,28 @@ def _find_non_coord_dimension_vars(group: H5Group) -> List[str]:
                     non_coordinate_dimension_variables.append(name)
 
         return non_coordinate_dimension_variables
+
+    @staticmethod
+    def _encode_cf_fill_value(
+        fill_value: Union[np.ndarray, np.generic],
+        target_dtype: np.dtype,
+    ) -> FillValueType:
+        """
+        Convert the _FillValue attribute from an HDF5 group or dataset into
+        one properly encoded for the target dtype.
+
+        Parameters
+        ----------
+        fill_value
+            An ndarray or value.
+        target_dtype
+            The target dtype of the ManifestArray that will use the _FillValue
+        """
+        if isinstance(fill_value, (np.ndarray, np.generic)):
+            if isinstance(fill_value, np.ndarray) and fill_value.size > 1:
+                raise ValueError("Expected a scalar")
+            fillvalue = fill_value.item()
+        else:
+            fillvalue = fill_value
+        encoded_fillvalue = FillValueCoder.encode(fillvalue, target_dtype)
+        return encoded_fillvalue
diff --git a/virtualizarr/tests/test_manifests/test_store.py b/virtualizarr/tests/test_manifests/test_store.py
index 094b003ac..c3617155b 100644
--- a/virtualizarr/tests/test_manifests/test_store.py
+++ b/virtualizarr/tests/test_manifests/test_store.py
@@ -140,6 +140,18 @@ def test_default_object_store_s3(minio_bucket):
     assert isinstance(store, S3Store)
 
 
+@requires_obstore
+@requires_minio
+def test_default_object_store_http(minio_bucket):
+    from obstore.store import HTTPStore
+
+    filepath = minio_bucket["endpoint"]
+    store = default_object_store(
+        filepath,
+    )
+    assert isinstance(store, HTTPStore)
+
+
 @requires_obstore
 def test_default_object_store_local(tmpdir):
     from obstore.store import LocalStore
diff --git a/virtualizarr/tests/test_readers/conftest.py b/virtualizarr/tests/test_readers/conftest.py
index d52b11b7d..81e6154fa 100644
--- a/virtualizarr/tests/test_readers/conftest.py
+++ b/virtualizarr/tests/test_readers/conftest.py
@@ -450,3 +450,11 @@ def chunked_roundtrip_hdf5_s3_file(minio_bucket, cf_array_fill_value_hdf5_file):
     filepath = "data/cf_array_fill_value.nc"
     obs.put(store, filepath, cf_array_fill_value_hdf5_file)
     return f"s3://{minio_bucket['bucket']}/{filepath}"
+
+
+@pytest.fixture()
+def big_endian_dtype_hdf5_file(tmpdir):
+    filepath = f"{tmpdir}/big_endian.nc"
+    f = h5py.File(filepath, "w")
+    f.create_dataset("data", shape=(10,), dtype=">f4")
+    return filepath
diff --git a/virtualizarr/tests/test_readers/test_hdf/test_hdf.py b/virtualizarr/tests/test_readers/test_hdf/test_hdf.py
index 5097fdd83..1cf84e731 100644
--- a/virtualizarr/tests/test_readers/test_hdf/test_hdf.py
+++ b/virtualizarr/tests/test_readers/test_hdf/test_hdf.py
@@ -1,6 +1,7 @@
 import h5py  # type: ignore
 import numpy as np
 import pytest
+from obstore.store import LocalStore
 
 from virtualizarr import open_virtual_dataset
 from virtualizarr.readers.hdf import HDFVirtualBackend
@@ -84,38 +85,38 @@ def test_no_dimension_scales(self, no_chunks_hdf5_file):
 
 @requires_hdf5plugin
 @requires_imagecodecs
-class TestDatasetToVariable:
+class TestDatasetToManifestArray:
     def test_chunked_dataset(self, chunked_dimensions_netcdf4_file):
         f = h5py.File(chunked_dimensions_netcdf4_file)
         ds = f["data"]
-        var = HDFVirtualBackend._dataset_to_variable(
+        ma = HDFVirtualBackend._construct_manifest_array(
             chunked_dimensions_netcdf4_file, ds, group=""
         )
-        assert var.chunks == (50, 50)
+        assert ma.chunks == (50, 50)
 
     def test_not_chunked_dataset(self, single_dimension_scale_hdf5_file):
         f = h5py.File(single_dimension_scale_hdf5_file)
         ds = f["data"]
-        var = HDFVirtualBackend._dataset_to_variable(
+        ma = HDFVirtualBackend._construct_manifest_array(
             single_dimension_scale_hdf5_file, ds, group=""
         )
-        assert var.chunks == (2,)
+        assert ma.chunks == (2,)
 
     def test_dataset_attributes(self, string_attributes_hdf5_file):
         f = h5py.File(string_attributes_hdf5_file)
         ds = f["data"]
-        var = HDFVirtualBackend._dataset_to_variable(
+        ma = HDFVirtualBackend._construct_manifest_array(
             string_attributes_hdf5_file, ds, group=""
         )
-        assert var.attrs["attribute_name"] == "attribute_name"
+        assert ma.metadata.attributes["attribute_name"] == "attribute_name"
 
     def test_scalar_fill_value(self, scalar_fill_value_hdf5_file):
         f = h5py.File(scalar_fill_value_hdf5_file)
         ds = f["data"]
-        var = HDFVirtualBackend._dataset_to_variable(
+        ma = HDFVirtualBackend._construct_manifest_array(
             scalar_fill_value_hdf5_file, ds, group=""
         )
-        assert var.data.metadata.fill_value == 42
+        assert ma.metadata.fill_value == 42
 
     def test_cf_fill_value(self, cf_fill_value_hdf5_file):
         f = h5py.File(cf_fill_value_hdf5_file)
@@ -126,18 +127,18 @@ def test_cf_fill_value(self, cf_fill_value_hdf5_file):
             pytest.xfail(
                 "To fix, structured dtype fill value encoding for Zarr backend"
             )
-        var = HDFVirtualBackend._dataset_to_variable(
+        ma = HDFVirtualBackend._construct_manifest_array(
             cf_fill_value_hdf5_file, ds, group=""
         )
-        assert "_FillValue" in var.encoding
+        assert "_FillValue" in ma.metadata.attributes
 
     def test_cf_array_fill_value(self, cf_array_fill_value_hdf5_file):
         f = h5py.File(cf_array_fill_value_hdf5_file)
         ds = f["data"]
-        var = HDFVirtualBackend._dataset_to_variable(
+        ma = HDFVirtualBackend._construct_manifest_array(
             cf_array_fill_value_hdf5_file, ds, group=""
         )
-        assert not isinstance(var.encoding["_FillValue"], np.ndarray)
+        assert not isinstance(ma.metadata.attributes["_FillValue"], np.ndarray)
 
 
 @requires_hdf5plugin
@@ -163,39 +164,52 @@ def test_multiple_attributes(self, string_attributes_hdf5_file):
 
 @requires_hdf5plugin
 @requires_imagecodecs
-class TestVirtualVarsFromHDF:
+class TestManifestGroupFromHDF:
     def test_variable_with_dimensions(self, chunked_dimensions_netcdf4_file):
-        variables = HDFVirtualBackend._virtual_vars_from_hdf(
-            chunked_dimensions_netcdf4_file
+        store = LocalStore()
+        manifest_group = HDFVirtualBackend._construct_manifest_group(
+            store=store,
+            filepath=chunked_dimensions_netcdf4_file,
         )
-        assert len(variables) == 3
+        assert len(manifest_group.arrays) == 3
 
     def test_nested_groups_are_ignored(self, nested_group_hdf5_file):
-        variables = HDFVirtualBackend._virtual_vars_from_hdf(
-            path=nested_group_hdf5_file, group="group"
+        store = LocalStore()
+        manifest_group = HDFVirtualBackend._construct_manifest_group(
+            store=store,
+            filepath=nested_group_hdf5_file,
+            group="group",
         )
-        assert len(variables) == 1
+        assert len(manifest_group.arrays) == 1
 
     def test_drop_variables(self, multiple_datasets_hdf5_file):
-        variables = HDFVirtualBackend._virtual_vars_from_hdf(
-            path=multiple_datasets_hdf5_file, drop_variables=["data2"]
+        store = LocalStore()
+        manifest_group = HDFVirtualBackend._construct_manifest_group(
+            store=store,
+            filepath=multiple_datasets_hdf5_file,
+            drop_variables=["data2"],
         )
-        assert "data2" not in variables.keys()
+        assert "data2" not in manifest_group.arrays.keys()
 
     def test_dataset_in_group(self, group_hdf5_file):
-        variables = HDFVirtualBackend._virtual_vars_from_hdf(
-            path=group_hdf5_file, group="group"
+        store = LocalStore()
+        manifest_group = HDFVirtualBackend._construct_manifest_group(
+            store=store,
+            filepath=group_hdf5_file,
+            group="group",
         )
-        assert len(variables) == 1
+        assert len(manifest_group.arrays) == 1
 
     def test_non_group_error(self, group_hdf5_file):
+        store = LocalStore()
         with pytest.raises(ValueError):
-            HDFVirtualBackend._virtual_vars_from_hdf(
-                path=group_hdf5_file, group="group/data"
+            HDFVirtualBackend._construct_manifest_group(
+                store=store,
+                filepath=group_hdf5_file,
+                group="group/data",
             )
 
 
-@pytest.mark.xfail(reason="no idea")
 @requires_hdf5plugin
 @requires_imagecodecs
 class TestOpenVirtualDataset:
@@ -206,6 +220,14 @@ def test_coord_names(
         vds = HDFVirtualBackend.open_virtual_dataset(root_coordinates_hdf5_file)
         assert set(vds.coords) == {"lat", "lon"}
 
+    @pytest.mark.xfail(reason="Requires Zarr v3 big endian dtype support")
+    def test_big_endian(
+        self,
+        big_endian_dtype_hdf5_file,
+    ):
+        vds = HDFVirtualBackend.open_virtual_dataset(big_endian_dtype_hdf5_file)
+        print(vds)
+
 
 @requires_hdf5plugin
 @requires_imagecodecs


Reviews:
user: login: TomNicholas
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-13T15:05:58Z
user: login: maxrjones
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-19T19:46:07Z
user: login: sharkinsspatial
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-19T19:58:59Z
user: login: sharkinsspatial
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-19T20:01:34Z
user: login: sharkinsspatial
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-19T23:16:21Z
user: login: sharkinsspatial
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-21T14:48:02Z
user: login: maxrjones
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-21T15:24:20Z
user: login: sharkinsspatial
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-21T16:02:36Z
user: login: sharkinsspatial
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-21T16:40:17Z

Review comments:
path: virtualizarr/common.py
user: login: TomNicholas
body: You shouldn't need this file anymore - it's all in `xarray.py` instead.
created_at: 2025-04-13T15:05:58Z
author_association: MEMBER
diff_hunk: @@ -50,6 +50,7 @@ hdf = [
     "hdf5plugin",
     "imagecodecs",
     "imagecodecs-numcodecs==2024.6.1",
+    "obstore",
path: pyproject.toml
user: login: maxrjones
body: ```suggestion
    "obstore>=0.5.1",
```
created_at: 2025-04-19T19:36:38Z
author_association: MEMBER
diff_hunk: @@ -177,9 +178,9 @@ run-mypy = { cmd = "mypy virtualizarr" }
 # is used), reduces test hangs that appear to be macOS-related.
 run-tests = { cmd = "pytest -n auto --dist loadscope --run-network-tests --verbose" }
 run-tests-no-network = { cmd = "pytest -n auto --verbose" }
-run-tests-cov = { cmd = "pytest -n auto --run-network-tests --verbose --cov=term-missing" }
-run-tests-xml-cov = { cmd = "pytest -n auto --run-network-tests --verbose --cov-report=xml" }
-run-tests-html-cov = { cmd = "pytest -n auto --run-network-tests --verbose --cov-report=html" }
+run-tests-cov = { cmd = "pytest -n 1 --run-network-tests --verbose --cov=virtualizarr --cov=term-missing" }
+run-tests-xml-cov = { cmd = "pytest -n 1 --run-network-tests --verbose --cov=virtualizarr --cov-report=xml" }
+run-tests-html-cov = { cmd = "pytest -n 1 --run-network-tests --verbose --cov=virtualizarr --cov-report=html" }
path: pyproject.toml
user: login: maxrjones
body: ```suggestion
run-tests-cov = { cmd = "pytest -n auto --run-network-tests --verbose --cov=term-missing" }
run-tests-xml-cov = { cmd = "pytest -n auto --run-network-tests --verbose --cov-report=xml" }
run-tests-html-cov = { cmd = "pytest -n auto --run-network-tests --verbose --cov-report=html" }
```
Not sure if you meant to include these changes?
created_at: 2025-04-19T19:37:17Z
author_association: MEMBER
diff_hunk: @@ -88,6 +81,22 @@ def _construct_manifest_array(
         attrs = HDFVirtualBackend._extract_attrs(dataset)
         dtype = dataset.dtype
 
+        # Temporarily disable use CF->Codecs - TODO re-enable in subsequent PR.
+        # cfcodec = cfcodec_from_dataset(dataset)
+        # if cfcodec:
+        # codecs.insert(0, cfcodec["codec"])
+        # dtype = cfcodec["target_dtype"]
+        # attrs.pop("scale_factor", None)
+        # attrs.pop("add_offset", None)
+        # else:
+        # dtype = dataset.dtype
path: virtualizarr/readers/hdf/hdf.py
user: login: maxrjones
body: What are the implications of disabling this? Will virtualization not work properly for datasets with a CF-style scale and offset?
created_at: 2025-04-19T19:39:05Z
author_association: MEMBER
diff_hunk: @@ -139,8 +148,8 @@ def _construct_manifest_group(
         non_coordinate_dimesion_vars = HDFVirtualBackend._find_non_coord_dimension_vars(
             group=g
         )
-        drop_variables = list(set(drop_variables + non_coordinate_dimesion_vars))
-        attrs: dict[str, Any] = {}
+        drop_variables = list(set(list(drop_variables) + non_coordinate_dimesion_vars))
+        attrs = HDFVirtualBackend._extract_attrs(g)
path: virtualizarr/readers/hdf/hdf.py
user: login: maxrjones
body: Can you add a comment for why `non_coordinate_dimension_vars` is added to `drop_variables`? Does the use of t`o_virtual_dataset` not help isolate this type of decision from the readers?
created_at: 2025-04-19T19:40:41Z
author_association: MEMBER
diff_hunk: @@ -139,8 +148,8 @@ def _construct_manifest_group(
         non_coordinate_dimesion_vars = HDFVirtualBackend._find_non_coord_dimension_vars(
path: virtualizarr/readers/hdf/hdf.py
user: login: maxrjones
body: ```suggestion
        non_coordinate_dimension_vars = HDFVirtualBackend._find_non_coord_dimension_vars(
```
created_at: 2025-04-19T19:40:56Z
author_association: MEMBER
diff_hunk: @@ -139,8 +148,8 @@ def _construct_manifest_group(
         non_coordinate_dimesion_vars = HDFVirtualBackend._find_non_coord_dimension_vars(
             group=g
         )
-        drop_variables = list(set(drop_variables + non_coordinate_dimesion_vars))
-        attrs: dict[str, Any] = {}
+        drop_variables = list(set(list(drop_variables) + non_coordinate_dimesion_vars))
path: virtualizarr/readers/hdf/hdf.py
user: login: maxrjones
body: ```suggestion
        drop_variables = list(set(list(drop_variables) + non_coordinate_dimension_vars))
```
created_at: 2025-04-19T19:41:01Z
author_association: MEMBER
diff_hunk: @@ -192,40 +205,19 @@ def open_virtual_dataset(
             filepath, fs_root=Path.cwd().as_uri()
         )
 
-        _drop_vars: list[Hashable] = (
+        _drop_vars: Iterable[str] = (
             [] if drop_variables is None else list(drop_variables)
         )
 
-        # TODO provide a way to drop a variable _before_ h5py attempts to inspect it?
-        virtual_vars = HDFVirtualBackend._virtual_vars_from_hdf(
-            path=filepath,
-            group=group,
-            reader_options=reader_options,
-        )
-
-        attrs = HDFVirtualBackend._get_group_attrs(
-            path=filepath, reader_options=reader_options, group=group
-        )
-        coordinates_attr = attrs.pop("coordinates", "")
-        coord_names = coordinates_attr.split()
-
-        fully_virtual_dataset = construct_fully_virtual_dataset(
-            virtual_vars=virtual_vars,
-            coord_names=coord_names,
-            attrs=attrs,
-        )
-
-        vds = construct_virtual_dataset(
-            fully_virtual_ds=fully_virtual_dataset,
+        manifest_store = HDFVirtualBackend._create_manifest_store(
             filepath=filepath,
+            drop_variables=_drop_vars,
             group=group,
-            loadable_variables=loadable_variables,
-            reader_options=reader_options,
-            indexes=indexes,
-            decode_times=decode_times,
         )
-
-        return vds.drop_vars(_drop_vars)
+        ds = manifest_store.to_virtual_dataset(
+            loadable_variables=loadable_variables, decode_times=decode_times
+        )
path: virtualizarr/readers/hdf/hdf.py
user: login: maxrjones
body: do you need to pass indexes in here?
created_at: 2025-04-19T19:42:11Z
author_association: MEMBER
diff_hunk: @@ -88,6 +81,22 @@ def _construct_manifest_array(
         attrs = HDFVirtualBackend._extract_attrs(dataset)
         dtype = dataset.dtype
 
+        # Temporarily disable use CF->Codecs - TODO re-enable in subsequent PR.
+        # cfcodec = cfcodec_from_dataset(dataset)
+        # if cfcodec:
+        # codecs.insert(0, cfcodec["codec"])
+        # dtype = cfcodec["target_dtype"]
+        # attrs.pop("scale_factor", None)
+        # attrs.pop("add_offset", None)
+        # else:
+        # dtype = dataset.dtype
path: virtualizarr/readers/hdf/hdf.py
user: login: sharkinsspatial
body: @maxrjones When I first created the reader I used this as a proof of concept to demonstrate using codecs for handling CF encoding to make data decoding more accessible for non-xarray clients https://github.com/zarr-developers/VirtualiZarr/issues/68#issuecomment-2048034811.  This works as expected but now causes failure of existing roundtrip tests now that we're not directly creating xarray structures.  To reduce the scope of the PR I've disabled it here (but the CF attrs are still carried through so all tests pass).  I'll re-enable this in a separate PR.
created_at: 2025-04-19T19:58:59Z
author_association: COLLABORATOR
diff_hunk: @@ -139,8 +148,8 @@ def _construct_manifest_group(
         non_coordinate_dimesion_vars = HDFVirtualBackend._find_non_coord_dimension_vars(
             group=g
         )
-        drop_variables = list(set(drop_variables + non_coordinate_dimesion_vars))
-        attrs: dict[str, Any] = {}
+        drop_variables = list(set(list(drop_variables) + non_coordinate_dimesion_vars))
+        attrs = HDFVirtualBackend._extract_attrs(g)
path: virtualizarr/readers/hdf/hdf.py
user: login: sharkinsspatial
body: @maxrjones 👍 I'll add a comment and an xfailing test.  This is just a temporary patch because some of our fixtures generated from xarray tutorial data serialize dimensions to HDF with big endian dtypes that are not yet supported in Zarr v3 https://github.com/zarr-developers/VirtualiZarr/pull/545
created_at: 2025-04-19T20:01:34Z
author_association: COLLABORATOR
diff_hunk: @@ -192,40 +205,19 @@ def open_virtual_dataset(
             filepath, fs_root=Path.cwd().as_uri()
         )
 
-        _drop_vars: list[Hashable] = (
+        _drop_vars: Iterable[str] = (
             [] if drop_variables is None else list(drop_variables)
         )
 
-        # TODO provide a way to drop a variable _before_ h5py attempts to inspect it?
-        virtual_vars = HDFVirtualBackend._virtual_vars_from_hdf(
-            path=filepath,
-            group=group,
-            reader_options=reader_options,
-        )
-
-        attrs = HDFVirtualBackend._get_group_attrs(
-            path=filepath, reader_options=reader_options, group=group
-        )
-        coordinates_attr = attrs.pop("coordinates", "")
-        coord_names = coordinates_attr.split()
-
-        fully_virtual_dataset = construct_fully_virtual_dataset(
-            virtual_vars=virtual_vars,
-            coord_names=coord_names,
-            attrs=attrs,
-        )
-
-        vds = construct_virtual_dataset(
-            fully_virtual_ds=fully_virtual_dataset,
+        manifest_store = HDFVirtualBackend._create_manifest_store(
             filepath=filepath,
+            drop_variables=_drop_vars,
             group=group,
-            loadable_variables=loadable_variables,
-            reader_options=reader_options,
-            indexes=indexes,
-            decode_times=decode_times,
         )
-
-        return vds.drop_vars(_drop_vars)
+        ds = manifest_store.to_virtual_dataset(
+            loadable_variables=loadable_variables, decode_times=decode_times
+        )
path: virtualizarr/readers/hdf/hdf.py
user: login: sharkinsspatial
body: IIUC we don't have support for processing indexes yet https://github.com/zarr-developers/VirtualiZarr/blob/develop/virtualizarr/xarray.py#L56-L57
created_at: 2025-04-19T23:16:21Z
author_association: COLLABORATOR
diff_hunk: @@ -139,8 +148,8 @@ def _construct_manifest_group(
         non_coordinate_dimesion_vars = HDFVirtualBackend._find_non_coord_dimension_vars(
             group=g
         )
-        drop_variables = list(set(drop_variables + non_coordinate_dimesion_vars))
-        attrs: dict[str, Any] = {}
+        drop_variables = list(set(list(drop_variables) + non_coordinate_dimesion_vars))
+        attrs = HDFVirtualBackend._extract_attrs(g)
path: virtualizarr/readers/hdf/hdf.py
user: login: sharkinsspatial
body: @maxrjones This commit should address this https://github.com/zarr-developers/VirtualiZarr/pull/542/commits/1d128394b408c034dc71a2cd60e160a1b0dd15dc
created_at: 2025-04-21T14:48:02Z
author_association: COLLABORATOR
diff_hunk: @@ -192,40 +205,19 @@ def open_virtual_dataset(
             filepath, fs_root=Path.cwd().as_uri()
         )
 
-        _drop_vars: list[Hashable] = (
+        _drop_vars: Iterable[str] = (
             [] if drop_variables is None else list(drop_variables)
         )
 
-        # TODO provide a way to drop a variable _before_ h5py attempts to inspect it?
-        virtual_vars = HDFVirtualBackend._virtual_vars_from_hdf(
-            path=filepath,
-            group=group,
-            reader_options=reader_options,
-        )
-
-        attrs = HDFVirtualBackend._get_group_attrs(
-            path=filepath, reader_options=reader_options, group=group
-        )
-        coordinates_attr = attrs.pop("coordinates", "")
-        coord_names = coordinates_attr.split()
-
-        fully_virtual_dataset = construct_fully_virtual_dataset(
-            virtual_vars=virtual_vars,
-            coord_names=coord_names,
-            attrs=attrs,
-        )
-
-        vds = construct_virtual_dataset(
-            fully_virtual_ds=fully_virtual_dataset,
+        manifest_store = HDFVirtualBackend._create_manifest_store(
             filepath=filepath,
+            drop_variables=_drop_vars,
             group=group,
-            loadable_variables=loadable_variables,
-            reader_options=reader_options,
-            indexes=indexes,
-            decode_times=decode_times,
         )
-
-        return vds.drop_vars(_drop_vars)
+        ds = manifest_store.to_virtual_dataset(
+            loadable_variables=loadable_variables, decode_times=decode_times
+        )
path: virtualizarr/readers/hdf/hdf.py
user: login: maxrjones
body: IMO it'd be better to raise the not supported error than the silently not do anything if a user passes in an argument to indexes
created_at: 2025-04-21T15:24:20Z
author_association: MEMBER
diff_hunk: @@ -192,40 +205,19 @@ def open_virtual_dataset(
             filepath, fs_root=Path.cwd().as_uri()
         )
 
-        _drop_vars: list[Hashable] = (
+        _drop_vars: Iterable[str] = (
             [] if drop_variables is None else list(drop_variables)
         )
 
-        # TODO provide a way to drop a variable _before_ h5py attempts to inspect it?
-        virtual_vars = HDFVirtualBackend._virtual_vars_from_hdf(
-            path=filepath,
-            group=group,
-            reader_options=reader_options,
-        )
-
-        attrs = HDFVirtualBackend._get_group_attrs(
-            path=filepath, reader_options=reader_options, group=group
-        )
-        coordinates_attr = attrs.pop("coordinates", "")
-        coord_names = coordinates_attr.split()
-
-        fully_virtual_dataset = construct_fully_virtual_dataset(
-            virtual_vars=virtual_vars,
-            coord_names=coord_names,
-            attrs=attrs,
-        )
-
-        vds = construct_virtual_dataset(
-            fully_virtual_ds=fully_virtual_dataset,
+        manifest_store = HDFVirtualBackend._create_manifest_store(
             filepath=filepath,
+            drop_variables=_drop_vars,
             group=group,
-            loadable_variables=loadable_variables,
-            reader_options=reader_options,
-            indexes=indexes,
-            decode_times=decode_times,
         )
-
-        return vds.drop_vars(_drop_vars)
+        ds = manifest_store.to_virtual_dataset(
+            loadable_variables=loadable_variables, decode_times=decode_times
+        )
path: virtualizarr/readers/hdf/hdf.py
user: login: sharkinsspatial
body: Agreed
created_at: 2025-04-21T16:02:36Z
author_association: COLLABORATOR
diff_hunk: @@ -192,40 +205,19 @@ def open_virtual_dataset(
             filepath, fs_root=Path.cwd().as_uri()
         )
 
-        _drop_vars: list[Hashable] = (
+        _drop_vars: Iterable[str] = (
             [] if drop_variables is None else list(drop_variables)
         )
 
-        # TODO provide a way to drop a variable _before_ h5py attempts to inspect it?
-        virtual_vars = HDFVirtualBackend._virtual_vars_from_hdf(
-            path=filepath,
-            group=group,
-            reader_options=reader_options,
-        )
-
-        attrs = HDFVirtualBackend._get_group_attrs(
-            path=filepath, reader_options=reader_options, group=group
-        )
-        coordinates_attr = attrs.pop("coordinates", "")
-        coord_names = coordinates_attr.split()
-
-        fully_virtual_dataset = construct_fully_virtual_dataset(
-            virtual_vars=virtual_vars,
-            coord_names=coord_names,
-            attrs=attrs,
-        )
-
-        vds = construct_virtual_dataset(
-            fully_virtual_ds=fully_virtual_dataset,
+        manifest_store = HDFVirtualBackend._create_manifest_store(
             filepath=filepath,
+            drop_variables=_drop_vars,
             group=group,
-            loadable_variables=loadable_variables,
-            reader_options=reader_options,
-            indexes=indexes,
-            decode_times=decode_times,
         )
-
-        return vds.drop_vars(_drop_vars)
+        ds = manifest_store.to_virtual_dataset(
+            loadable_variables=loadable_variables, decode_times=decode_times
+        )
path: virtualizarr/readers/hdf/hdf.py
user: login: sharkinsspatial
body: @maxrjones Addressed by https://github.com/zarr-developers/VirtualiZarr/pull/542/commits/a3bfca98faf405acdc5982517fa172c232d85d03
created_at: 2025-04-21T16:40:16Z
author_association: COLLABORATOR

PR comments:
user: login: TomNicholas
created_at: 2025-04-10T17:16:53Z
body: @sharkinsspatial I think I've been refactoring the exact same bit of code locally 😅
author_association: MEMBER
user: login: TomNicholas
created_at: 2025-04-10T23:04:07Z
body: Use this PR instead @sharkinsspatial 

https://github.com/zarr-developers/VirtualiZarr/pull/543
author_association: MEMBER
user: login: TomNicholas
created_at: 2025-04-11T22:06:05Z
body: Why is this diff now 1000 lines?
author_association: MEMBER
user: login: sharkinsspatial
created_at: 2025-04-11T22:09:36Z
body: 🤦 Uff. I seemed to rebase something incorrectly here.
author_association: COLLABORATOR
user: login: codecov[bot]
created_at: 2025-04-19T18:01:51Z
body: ## [Codecov](https://app.codecov.io/gh/zarr-developers/VirtualiZarr/pull/542?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=zarr-developers) Report
Attention: Patch coverage is `87.50000%` with `3 lines` in your changes missing coverage. Please review.
> Project coverage is 87.28%. Comparing base [(`fd2ccf3`)](https://app.codecov.io/gh/zarr-developers/VirtualiZarr/commit/fd2ccf311364e5deee2af3ae4d1bade12e3b1004?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=zarr-developers) to head [(`2ea03fa`)](https://app.codecov.io/gh/zarr-developers/VirtualiZarr/commit/2ea03faca5a4573044e46a8744eea74d0d4a9778?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=zarr-developers).
> Report is 22 commits behind head on develop.

| [Files with missing lines](https://app.codecov.io/gh/zarr-developers/VirtualiZarr/pull/542?dropdown=coverage&src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=zarr-developers) | Patch % | Lines |
|---|---|---|
| [virtualizarr/readers/hdf/hdf.py](https://app.codecov.io/gh/zarr-developers/VirtualiZarr/pull/542?src=pr&el=tree&filepath=virtualizarr%2Freaders%2Fhdf%2Fhdf.py&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=zarr-developers#diff-dmlydHVhbGl6YXJyL3JlYWRlcnMvaGRmL2hkZi5weQ==) | 85.71% | [3 Missing :warning: ](https://app.codecov.io/gh/zarr-developers/VirtualiZarr/pull/542?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=zarr-developers) |

<details><summary>Additional details and impacted files</summary>


```diff
@@             Coverage Diff             @@
##           develop     #542      +/-   ##
===========================================
- Coverage    87.67%   87.28%   -0.39%     
===========================================
  Files           31       32       +1     
  Lines         1890     2092     +202     
===========================================
+ Hits          1657     1826     +169     
- Misses         233      266      +33     
```

| [Files with missing lines](https://app.codecov.io/gh/zarr-developers/VirtualiZarr/pull/542?dropdown=coverage&src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=zarr-developers) | Coverage Δ | |
|---|---|---|
| [virtualizarr/manifests/store.py](https://app.codecov.io/gh/zarr-developers/VirtualiZarr/pull/542?src=pr&el=tree&filepath=virtualizarr%2Fmanifests%2Fstore.py&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=zarr-developers#diff-dmlydHVhbGl6YXJyL21hbmlmZXN0cy9zdG9yZS5weQ==) | `91.56% <100.00%> (+6.14%)` | :arrow_up: |
| [virtualizarr/readers/hdf/hdf.py](https://app.codecov.io/gh/zarr-developers/VirtualiZarr/pull/542?src=pr&el=tree&filepath=virtualizarr%2Freaders%2Fhdf%2Fhdf.py&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=zarr-developers#diff-dmlydHVhbGl6YXJyL3JlYWRlcnMvaGRmL2hkZi5weQ==) | `94.25% <85.71%> (-0.68%)` | :arrow_down: |

... and [10 files with indirect coverage changes](https://app.codecov.io/gh/zarr-developers/VirtualiZarr/pull/542/indirect-changes?src=pr&el=tree-more&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=zarr-developers)

</details>
<details><summary> :rocket: New features to boost your workflow: </summary>

- :snowflake: [Test Analytics](https://docs.codecov.com/docs/test-analytics): Detect flaky tests, report on failures, and find test suite problems.
</details>
author_association: NONE