Repository: zarr-developers/zarr-python

Issue:
Fix fill_value encoding for fixed-width string / binary v2 array
### Zarr version

v3

### Numcodecs version

na

### Python Version

na

### Operating System

na

### Installation

na

### Description

In https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html#fill-value-encoding, the zarr-v2 spec says that `fill_value` must be base64 encoded for some dtypes:

> If an array has a fixed length byte string data type (e.g., "|S12"), or a structured data type, and if the fill value is not null, then the fill value MUST be encoded as an ASCII string using the standard Base64 alphabet.

### Steps to reproduce

Given this

```
    store = zarr.storage.MemoryStore(mode="w")
    g = zarr.group(store=store, zarr_format=2)
    g.create_array(
        name="foo",
        shape=(3,),
        dtype="|S4",
        fill_value=b"X",
    )
```

we produce metadata like this currently:

```
 {'chunks': [3],
 'compressor': None,
 'dimension_separator': '.',
 'dtype': '|S4',
 'fill_value': [88],
 'filters': None,
 'order': 'C',
 'shape': [3],
 'zarr_format': 2}
```

It should be something like

```
{'chunks': [3],
 'compressor': {'blocksize': 0,
                'clevel': 5,
                'cname': 'lz4',
                'id': 'blosc',
                'shuffle': 1},
 'dtype': '|S4',
 'fill_value': 'WA==',
 'filters': None,
 'order': 'C',
 'shape': [3],
 'zarr_format': 2}
```

Note that `WA==` is the ascii representation of the base64encoded value `X`.

### Additional output

_No response_

Discussion:
TomAugspurger: This should be ready to go.
d-v-b: were we not doing this in the v2 codebase? I couldn't find code for it.
TomAugspurger: I found it at https://github.com/zarr-developers/zarr-python/blob/6b9ab9e29f526986c5c7519dd4063546d5a81d14/zarr/meta.py#L215 (and borrowed the implementation).
jakirkham: Thanks Tom! 🙏

JFYI we have [a `Base64` codec in Numcodecs]( https://github.com/zarr-developers/numcodecs/blob/71fcab086ba3b8efb4c886fbaba22d9dcc55df62/numcodecs/base64.py#L7 ), perhaps it is worth using to align on behavior

Patch:
diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml
index b03ea06b..f694039c 100644
--- a/.github/workflows/releases.yml
+++ b/.github/workflows/releases.yml
@@ -55,7 +55,7 @@ jobs:
         with:
           name: releases
           path: dist
-      - uses: pypa/gh-action-pypi-publish@v1.10.2
+      - uses: pypa/gh-action-pypi-publish@v1.10.3
         with:
           user: __token__
           password: ${{ secrets.pypi_password }}
diff --git a/src/zarr/abc/codec.py b/src/zarr/abc/codec.py
index afe3e34b..83e89d14 100644
--- a/src/zarr/abc/codec.py
+++ b/src/zarr/abc/codec.py
@@ -20,6 +20,7 @@ if TYPE_CHECKING:
     from zarr.core.indexing import SelectorTuple
 
 __all__ = [
+    "BaseCodec",
     "ArrayArrayCodec",
     "ArrayBytesCodec",
     "ArrayBytesCodecPartialDecodeMixin",
@@ -34,11 +35,15 @@ CodecInput = TypeVar("CodecInput", bound=NDBuffer | Buffer)
 CodecOutput = TypeVar("CodecOutput", bound=NDBuffer | Buffer)
 
 
-class _Codec(Metadata, Generic[CodecInput, CodecOutput]):
+class BaseCodec(Metadata, Generic[CodecInput, CodecOutput]):
     """Generic base class for codecs.
-    Please use ArrayArrayCodec, ArrayBytesCodec or BytesBytesCodec for subclassing.
 
     Codecs can be registered via zarr.codecs.registry.
+
+    Warnings
+    --------
+    This class is not intended to be directly, please use
+    ArrayArrayCodec, ArrayBytesCodec or BytesBytesCodec for subclassing.
     """
 
     is_fixed_size: bool
@@ -148,19 +153,19 @@ class _Codec(Metadata, Generic[CodecInput, CodecOutput]):
         return await _batching_helper(self._encode_single, chunks_and_specs)
 
 
-class ArrayArrayCodec(_Codec[NDBuffer, NDBuffer]):
+class ArrayArrayCodec(BaseCodec[NDBuffer, NDBuffer]):
     """Base class for array-to-array codecs."""
 
     ...
 
 
-class ArrayBytesCodec(_Codec[NDBuffer, Buffer]):
+class ArrayBytesCodec(BaseCodec[NDBuffer, Buffer]):
     """Base class for array-to-bytes codecs."""
 
     ...
 
 
-class BytesBytesCodec(_Codec[Buffer, Buffer]):
+class BytesBytesCodec(BaseCodec[Buffer, Buffer]):
     """Base class for bytes-to-bytes codecs."""
 
     ...
diff --git a/src/zarr/abc/metadata.py b/src/zarr/abc/metadata.py
index 239d151c..291ceb45 100644
--- a/src/zarr/abc/metadata.py
+++ b/src/zarr/abc/metadata.py
@@ -22,7 +22,6 @@ class Metadata:
         are instances of `Metadata`. Sequences of `Metadata` are similarly recursed into, and
         the output of that recursion is collected in a list.
         """
-        ...
         out_dict = {}
         for field in fields(self):
             key = field.name
diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py
index 08ac7665..adcaf46e 100644
--- a/src/zarr/core/group.py
+++ b/src/zarr/core/group.py
@@ -768,24 +768,20 @@ class AsyncGroup:
         )
 
     async def empty_like(
-        self, *, name: str, prototype: async_api.ArrayLike, **kwargs: Any
+        self, *, name: str, data: async_api.ArrayLike, **kwargs: Any
     ) -> AsyncArray:
-        return await async_api.empty_like(a=prototype, store=self.store_path, path=name, **kwargs)
+        return await async_api.empty_like(a=data, store=self.store_path, path=name, **kwargs)
 
     async def zeros_like(
-        self, *, name: str, prototype: async_api.ArrayLike, **kwargs: Any
+        self, *, name: str, data: async_api.ArrayLike, **kwargs: Any
     ) -> AsyncArray:
-        return await async_api.zeros_like(a=prototype, store=self.store_path, path=name, **kwargs)
+        return await async_api.zeros_like(a=data, store=self.store_path, path=name, **kwargs)
 
-    async def ones_like(
-        self, *, name: str, prototype: async_api.ArrayLike, **kwargs: Any
-    ) -> AsyncArray:
-        return await async_api.ones_like(a=prototype, store=self.store_path, path=name, **kwargs)
+    async def ones_like(self, *, name: str, data: async_api.ArrayLike, **kwargs: Any) -> AsyncArray:
+        return await async_api.ones_like(a=data, store=self.store_path, path=name, **kwargs)
 
-    async def full_like(
-        self, *, name: str, prototype: async_api.ArrayLike, **kwargs: Any
-    ) -> AsyncArray:
-        return await async_api.full_like(a=prototype, store=self.store_path, path=name, **kwargs)
+    async def full_like(self, *, name: str, data: async_api.ArrayLike, **kwargs: Any) -> AsyncArray:
+        return await async_api.full_like(a=data, store=self.store_path, path=name, **kwargs)
 
     async def move(self, source: str, dest: str) -> None:
         raise NotImplementedError
@@ -1171,25 +1167,17 @@ class Group(SyncMixin):
             )
         )
 
-    def empty_like(self, *, name: str, prototype: async_api.ArrayLike, **kwargs: Any) -> Array:
-        return Array(
-            self._sync(self._async_group.empty_like(name=name, prototype=prototype, **kwargs))
-        )
+    def empty_like(self, *, name: str, data: async_api.ArrayLike, **kwargs: Any) -> Array:
+        return Array(self._sync(self._async_group.empty_like(name=name, data=data, **kwargs)))
 
-    def zeros_like(self, *, name: str, prototype: async_api.ArrayLike, **kwargs: Any) -> Array:
-        return Array(
-            self._sync(self._async_group.zeros_like(name=name, prototype=prototype, **kwargs))
-        )
+    def zeros_like(self, *, name: str, data: async_api.ArrayLike, **kwargs: Any) -> Array:
+        return Array(self._sync(self._async_group.zeros_like(name=name, data=data, **kwargs)))
 
-    def ones_like(self, *, name: str, prototype: async_api.ArrayLike, **kwargs: Any) -> Array:
-        return Array(
-            self._sync(self._async_group.ones_like(name=name, prototype=prototype, **kwargs))
-        )
+    def ones_like(self, *, name: str, data: async_api.ArrayLike, **kwargs: Any) -> Array:
+        return Array(self._sync(self._async_group.ones_like(name=name, data=data, **kwargs)))
 
-    def full_like(self, *, name: str, prototype: async_api.ArrayLike, **kwargs: Any) -> Array:
-        return Array(
-            self._sync(self._async_group.full_like(name=name, prototype=prototype, **kwargs))
-        )
+    def full_like(self, *, name: str, data: async_api.ArrayLike, **kwargs: Any) -> Array:
+        return Array(self._sync(self._async_group.full_like(name=name, data=data, **kwargs)))
 
     def move(self, source: str, dest: str) -> None:
         return self._sync(self._async_group.move(source, dest))
diff --git a/src/zarr/core/metadata/v2.py b/src/zarr/core/metadata/v2.py
index 19b44ec4..ec44673f 100644
--- a/src/zarr/core/metadata/v2.py
+++ b/src/zarr/core/metadata/v2.py
@@ -1,8 +1,9 @@
 from __future__ import annotations
 
+import base64
 from collections.abc import Iterable
 from enum import Enum
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, cast
 
 if TYPE_CHECKING:
     from typing import Any, Literal, Self
@@ -31,7 +32,7 @@ class ArrayV2Metadata(ArrayMetadata):
     shape: ChunkCoords
     chunk_grid: RegularChunkGrid
     data_type: np.dtype[Any]
-    fill_value: None | int | float = 0
+    fill_value: None | int | float | str | bytes = 0
     order: Literal["C", "F"] = "C"
     filters: tuple[numcodecs.abc.Codec, ...] | None = None
     dimension_separator: Literal[".", "/"] = "."
@@ -140,6 +141,13 @@ class ArrayV2Metadata(ArrayMetadata):
         _data = data.copy()
         # check that the zarr_format attribute is correct
         _ = parse_zarr_format(_data.pop("zarr_format"))
+        dtype = parse_dtype(_data["dtype"])
+
+        if dtype.kind in "SV":
+            fill_value_encoded = _data.get("fill_value")
+            if fill_value_encoded is not None:
+                fill_value = base64.standard_b64decode(fill_value_encoded)
+                _data["fill_value"] = fill_value
 
         # zarr v2 allowed arbitrary keys here.
         # We don't want the ArrayV2Metadata constructor to fail just because someone put an
@@ -155,6 +163,14 @@ class ArrayV2Metadata(ArrayMetadata):
 
     def to_dict(self) -> dict[str, JSON]:
         zarray_dict = super().to_dict()
+
+        if self.dtype.kind in "SV" and self.fill_value is not None:
+            # There's a relationship between self.dtype and self.fill_value
+            # that mypy isn't aware of. The fact that we have S or V dtype here
+            # means we should have a bytes-type fill_value.
+            fill_value = base64.standard_b64encode(cast(bytes, self.fill_value)).decode("ascii")
+            zarray_dict["fill_value"] = fill_value
+
         _ = zarray_dict.pop("chunk_grid")
         zarray_dict["chunks"] = self.chunk_grid.chunk_shape
 


Test patch:
diff --git a/tests/v3/test_group.py b/tests/v3/test_group.py
index 9d9f4860..d41955b2 100644
--- a/tests/v3/test_group.py
+++ b/tests/v3/test_group.py
@@ -447,7 +447,7 @@ def test_group_array_creation(
     assert empty_array.shape == shape
     assert empty_array.store_path.store == store
 
-    empty_like_array = group.empty_like(name="empty_like", prototype=empty_array)
+    empty_like_array = group.empty_like(name="empty_like", data=empty_array)
     assert isinstance(empty_like_array, Array)
     assert empty_like_array.fill_value == 0
     assert empty_like_array.shape == shape
@@ -459,7 +459,7 @@ def test_group_array_creation(
     assert empty_array_bool.shape == shape
     assert empty_array_bool.store_path.store == store
 
-    empty_like_array_bool = group.empty_like(name="empty_like_bool", prototype=empty_array_bool)
+    empty_like_array_bool = group.empty_like(name="empty_like_bool", data=empty_array_bool)
     assert isinstance(empty_like_array_bool, Array)
     assert not empty_like_array_bool.fill_value
     assert empty_like_array_bool.shape == shape
@@ -471,7 +471,7 @@ def test_group_array_creation(
     assert zeros_array.shape == shape
     assert zeros_array.store_path.store == store
 
-    zeros_like_array = group.zeros_like(name="zeros_like", prototype=zeros_array)
+    zeros_like_array = group.zeros_like(name="zeros_like", data=zeros_array)
     assert isinstance(zeros_like_array, Array)
     assert zeros_like_array.fill_value == 0
     assert zeros_like_array.shape == shape
@@ -483,7 +483,7 @@ def test_group_array_creation(
     assert ones_array.shape == shape
     assert ones_array.store_path.store == store
 
-    ones_like_array = group.ones_like(name="ones_like", prototype=ones_array)
+    ones_like_array = group.ones_like(name="ones_like", data=ones_array)
     assert isinstance(ones_like_array, Array)
     assert ones_like_array.fill_value == 1
     assert ones_like_array.shape == shape
@@ -495,7 +495,7 @@ def test_group_array_creation(
     assert full_array.shape == shape
     assert full_array.store_path.store == store
 
-    full_like_array = group.full_like(name="full_like", prototype=full_array, fill_value=43)
+    full_like_array = group.full_like(name="full_like", data=full_array, fill_value=43)
     assert isinstance(full_like_array, Array)
     assert full_like_array.fill_value == 43
     assert full_like_array.shape == shape
diff --git a/tests/v3/test_v2.py b/tests/v3/test_v2.py
index 95a5f666..f488782d 100644
--- a/tests/v3/test_v2.py
+++ b/tests/v3/test_v2.py
@@ -1,3 +1,4 @@
+import json
 from collections.abc import Iterator
 
 import numpy as np
@@ -6,6 +7,9 @@ from numcodecs import Delta
 from numcodecs.blosc import Blosc
 
 import zarr
+import zarr.core.buffer.cpu
+import zarr.core.metadata
+import zarr.storage
 from zarr import Array
 from zarr.storage import MemoryStore, StorePath
 
@@ -46,3 +50,37 @@ def test_codec_pipeline() -> None:
     result = array[:]
     expected = np.ones(1)
     np.testing.assert_array_equal(result, expected)
+
+
+@pytest.mark.parametrize("dtype", ["|S", "|V"])
+async def test_v2_encode_decode(dtype):
+    store = zarr.storage.MemoryStore(mode="w")
+    g = zarr.group(store=store, zarr_format=2)
+    g.create_array(
+        name="foo",
+        shape=(3,),
+        chunks=(3,),
+        dtype=dtype,
+        fill_value=b"X",
+    )
+
+    result = await store.get("foo/.zarray", zarr.core.buffer.default_buffer_prototype())
+    assert result is not None
+
+    serialized = json.loads(result.to_bytes())
+    expected = {
+        "chunks": [3],
+        "compressor": None,
+        "dtype": f"{dtype}0",
+        "fill_value": "WA==",
+        "filters": None,
+        "order": "C",
+        "shape": [3],
+        "zarr_format": 2,
+        "dimension_separator": ".",
+    }
+    assert serialized == expected
+
+    data = zarr.open_array(store=store, path="foo")[:]
+    expected = np.full((3,), b"X", dtype=dtype)
+    np.testing.assert_equal(data, expected)