Repository: zarr-developers/zarr-python

Issue:
Chunk padding is non-deterministic with `zarr_format=2`
### Zarr version

3.0.0

### Numcodecs version

0.14.0

### Python Version

3.11

### Operating System

Mac

### Installation

pip

### Description

When using `zarr_format=2` with Zarr Python 3.0.0 it's possible to write chunks that have a different size each time they are written (even with the same data). This doesn't occur if `zarr_format=3`, or with Zarr Python 2.

The problem seems to be that the chunk is initialized with no fill value (using `np.empty`).

This was found in https://github.com/sgkit-dev/bio2zarr/pull/288#issuecomment-2575426493.

### Steps to reproduce

```python
import zarr

def run():
    store = zarr.storage.MemoryStore()
    a = zarr.empty(
        store=store,
        shape=(3,),
        chunks=(1000,),
        dtype=int,
        zarr_format=2,
    )
    a[:] = 1
    print(len(store._store_dict["0"]))

run()
run()
```

Example output (changes from run to run):
```
1890
2954
```

Changing to `zarr_format=3`, and the key to `"c/0"` gives the following consistently:

```
27
27
```

### Additional output

_No response_

Discussion:
normanrz: > Marking this as request changes, since it looks like it broke some tests - I had a look at the tests, and failure seems real (requesting a fill value of `""`, and getting `0` instead)

Fixed it.
normanrz: Both `zarr.empty` and `zarr.zeros` should be deprecated in favor of `zarr.create_array`.

The old docs might have said that `zarr.empty` returns unitialized data, but I don't think that was the case in the code. Here, entire are filled with `fill_value` even if outside the to-be-set selection: https://github.com/zarr-developers/zarr-python/pull/2755/files#diff-44efa3ae220ba9737fa8c5443b3c3af09a7c0f9549d5c267404f7fa3ce467318
d-v-b: i had a quick look through the v2 code and it looks like v2 was padding with 0s to ensure consistency and compressibility: https://github.com/zarr-developers/zarr-python/blob/66e2982d4d16ace4e32bf0deba627357c8982844/zarr/core.py#L2309-L2313
d-v-b: @brokkoli71 is this ready to go in?
brokkoli71: > @brokkoli71 is this ready to go in?

yes 👍🏼 @d-v-b 

Patch:
diff --git a/changes/2755.bugfix.rst b/changes/2755.bugfix.rst
new file mode 100644
index 00000000..25553695
--- /dev/null
+++ b/changes/2755.bugfix.rst
@@ -0,0 +1,3 @@
+The array returned by ``zarr.empty`` and an empty ``zarr.core.buffer.cpu.NDBuffer`` will now be filled with the
+specified fill value, or with zeros if no fill value is provided.
+This fixes a bug where Zarr format 2 data with no fill value was written with un-predictable chunk sizes.
\ No newline at end of file
diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py
index 8eba4fc1..0584f19c 100644
--- a/src/zarr/api/asynchronous.py
+++ b/src/zarr/api/asynchronous.py
@@ -1065,7 +1065,8 @@ async def create(
 async def empty(
     shape: ChunkCoords, **kwargs: Any
 ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]:
-    """Create an empty array.
+    """Create an empty array with the specified shape. The contents will be filled with the
+    array's fill value or zeros if no fill value is provided.
 
     Parameters
     ----------
@@ -1087,7 +1088,8 @@ async def empty(
 async def empty_like(
     a: ArrayLike, **kwargs: Any
 ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]:
-    """Create an empty array like `a`.
+    """Create an empty array like `a`. The contents will be filled with the
+    array's fill value or zeros if no fill value is provided.
 
     Parameters
     ----------
@@ -1100,6 +1102,12 @@ async def empty_like(
     -------
     Array
         The new array.
+
+    Notes
+    -----
+    The contents of an empty Zarr array are not defined. On attempting to
+    retrieve data from an empty Zarr array, any values may be returned,
+    and these are not guaranteed to be stable from one access to the next.
     """
     like_kwargs = _like_args(a, kwargs)
     return await empty(**like_kwargs)
diff --git a/src/zarr/api/synchronous.py b/src/zarr/api/synchronous.py
index 305446ec..fe68981c 100644
--- a/src/zarr/api/synchronous.py
+++ b/src/zarr/api/synchronous.py
@@ -902,7 +902,8 @@ def create_array(
 
 # TODO: add type annotations for kwargs
 def empty(shape: ChunkCoords, **kwargs: Any) -> Array:
-    """Create an empty array.
+    """Create an empty array with the specified shape. The contents will be filled with the
+    array's fill value or zeros if no fill value is provided.
 
     Parameters
     ----------
@@ -928,7 +929,8 @@ def empty(shape: ChunkCoords, **kwargs: Any) -> Array:
 # TODO: move ArrayLike to common module
 # TODO: add type annotations for kwargs
 def empty_like(a: ArrayLike, **kwargs: Any) -> Array:
-    """Create an empty array like another array.
+    """Create an empty array like another array. The contents will be filled with the
+    array's fill value or zeros if no fill value is provided.
 
     Parameters
     ----------
@@ -941,6 +943,12 @@ def empty_like(a: ArrayLike, **kwargs: Any) -> Array:
     -------
     Array
         The new array.
+
+    Notes
+    -----
+    The contents of an empty Zarr array are not defined. On attempting to
+    retrieve data from an empty Zarr array, any values may be returned,
+    and these are not guaranteed to be stable from one access to the next.
     """
     return Array(sync(async_api.empty_like(a, **kwargs)))
 
diff --git a/src/zarr/core/buffer/cpu.py b/src/zarr/core/buffer/cpu.py
index 50190754..225adb6f 100644
--- a/src/zarr/core/buffer/cpu.py
+++ b/src/zarr/core/buffer/cpu.py
@@ -154,10 +154,10 @@ class NDBuffer(core.NDBuffer):
         order: Literal["C", "F"] = "C",
         fill_value: Any | None = None,
     ) -> Self:
-        ret = cls(np.empty(shape=tuple(shape), dtype=dtype, order=order))
-        if fill_value is not None:
-            ret.fill(fill_value)
-        return ret
+        if fill_value is None:
+            return cls(np.zeros(shape=tuple(shape), dtype=dtype, order=order))
+        else:
+            return cls(np.full(shape=tuple(shape), fill_value=fill_value, dtype=dtype, order=order))
 
     @classmethod
     def from_numpy_array(cls, array_like: npt.ArrayLike) -> Self:
diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py
index 4760923e..1f5d57c0 100644
--- a/src/zarr/core/group.py
+++ b/src/zarr/core/group.py
@@ -1498,7 +1498,8 @@ class AsyncGroup:
     async def empty(
         self, *, name: str, shape: ChunkCoords, **kwargs: Any
     ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]:
-        """Create an empty array in this Group.
+        """Create an empty array with the specified shape in this Group. The contents will
+        be filled with the array's fill value or zeros if no fill value is provided.
 
         Parameters
         ----------
@@ -1515,7 +1516,6 @@ class AsyncGroup:
         retrieve data from an empty Zarr array, any values may be returned,
         and these are not guaranteed to be stable from one access to the next.
         """
-
         return await async_api.empty(shape=shape, store=self.store_path, path=name, **kwargs)
 
     async def zeros(
@@ -1592,7 +1592,8 @@ class AsyncGroup:
     async def empty_like(
         self, *, name: str, data: async_api.ArrayLike, **kwargs: Any
     ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]:
-        """Create an empty sub-array like `data`.
+        """Create an empty sub-array like `data`. The contents will be filled with
+        the array's fill value or zeros if no fill value is provided.
 
         Parameters
         ----------
@@ -2442,7 +2443,8 @@ class Group(SyncMixin):
 
     @_deprecate_positional_args
     def empty(self, *, name: str, shape: ChunkCoords, **kwargs: Any) -> Array:
-        """Create an empty array in this Group.
+        """Create an empty array with the specified shape in this Group. The contents will be filled with
+        the array's fill value or zeros if no fill value is provided.
 
         Parameters
         ----------
@@ -2531,7 +2533,8 @@ class Group(SyncMixin):
 
     @_deprecate_positional_args
     def empty_like(self, *, name: str, data: async_api.ArrayLike, **kwargs: Any) -> Array:
-        """Create an empty sub-array like `data`.
+        """Create an empty sub-array like `data`. The contents will be filled
+        with the array's fill value or zeros if no fill value is provided.
 
         Parameters
         ----------
@@ -2546,6 +2549,12 @@ class Group(SyncMixin):
         -------
         Array
             The new array.
+
+        Notes
+        -----
+        The contents of an empty Zarr array are not defined. On attempting to
+        retrieve data from an empty Zarr array, any values may be returned,
+        and these are not guaranteed to be stable from one access to the next.
         """
         return Array(self._sync(self._async_group.empty_like(name=name, data=data, **kwargs)))
 


Test patch:
diff --git a/tests/test_store/test_memory.py b/tests/test_store/test_memory.py
index ba38889b..f00d75a8 100644
--- a/tests/test_store/test_memory.py
+++ b/tests/test_store/test_memory.py
@@ -1,12 +1,19 @@
 from __future__ import annotations
 
+from typing import TYPE_CHECKING
+
+import numpy as np
 import pytest
 
+import zarr
 from zarr.core.buffer import Buffer, cpu, gpu
 from zarr.storage import GpuMemoryStore, MemoryStore
 from zarr.testing.store import StoreTests
 from zarr.testing.utils import gpu_test
 
+if TYPE_CHECKING:
+    from zarr.core.common import ZarrFormat
+
 
 class TestMemoryStore(StoreTests[MemoryStore, cpu.Buffer]):
     store_cls = MemoryStore
@@ -46,6 +53,25 @@ class TestMemoryStore(StoreTests[MemoryStore, cpu.Buffer]):
     def test_list_prefix(self, store: MemoryStore) -> None:
         assert True
 
+    @pytest.mark.parametrize("dtype", ["uint8", "float32", "int64"])
+    @pytest.mark.parametrize("zarr_format", [2, 3])
+    async def test_deterministic_size(
+        self, store: MemoryStore, dtype, zarr_format: ZarrFormat
+    ) -> None:
+        a = zarr.empty(
+            store=store,
+            shape=(3,),
+            chunks=(1000,),
+            dtype=dtype,
+            zarr_format=zarr_format,
+            overwrite=True,
+        )
+        a[...] = 1
+        a.resize((1000,))
+
+        np.testing.assert_array_equal(a[:3], 1)
+        np.testing.assert_array_equal(a[3:], 0)
+
 
 @gpu_test
 class TestGpuMemoryStore(StoreTests[GpuMemoryStore, gpu.Buffer]):